Disaster Recovery for Cloud Development Environments

Continuous Replication

# AWS RDS - Enable Multi-AZ + Read Replica
aws rds create-db-instance-read-replica \
  --db-instance-identifier coder-db-replica-eu \
  --source-db-instance-identifier coder-db-us \
  --source-region us-east-1 \
  --region eu-west-1

# PostgreSQL Logical Replication
# On primary
CREATE PUBLICATION coder_pub FOR ALL TABLES;

# On replica
CREATE SUBSCRIPTION coder_sub
CONNECTION 'host=primary-db port=5432 dbname=coder'
PUBLICATION coder_pub;

Automated Snapshots

# Enable automated backups
aws rds modify-db-instance \
  --db-instance-identifier coder-db \
  --backup-retention-period 30 \
  --preferred-backup-window "03:00-04:00"

# Manual snapshot for testing
aws rds create-db-snapshot \
  --db-instance-identifier coder-db \
  --db-snapshot-identifier pre-upgrade-$(date +%Y%m%d)

# Copy snapshot to DR region
aws rds copy-db-snapshot \
  --source-region us-east-1 \
  --source-db-snapshot-identifier SNAPSHOT_ID \
  --target-db-snapshot-identifier SNAPSHOT_ID \
  --region eu-west-1

Volume Snapshots

# Kubernetes VolumeSnapshot (CSI)
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshot
metadata:
  name: workspace-snapshot
spec:
  volumeSnapshotClassName: csi-aws-vsc
  source:
    persistentVolumeClaimName: workspace-pvc

# Automated snapshot via CronJob
kubectl create cronjob workspace-snapshots \
  --image=bitnami/kubectl \
  --schedule="0 */4 * * *" \
  -- kubectl create volumesnapshot \
  workspace-$(date +%Y%m%d-%H%M) \
  --from-pvc=workspace-pvc

Selective Backup with Velero

# Install Velero with S3 backend
velero install \
  --provider aws \
  --bucket cde-backups \
  --backup-location-config region=us-east-1 \
  --snapshot-location-config region=us-east-1

# Create backup schedule
velero schedule create daily-workspaces \
  --schedule="0 2 * * *" \
  --include-namespaces workspaces \
  --ttl 720h

# Backup critical workspace only
velero backup create important-workspace \
  --include-resources pvc \
  --selector workspace=prod-db-workspace

Git-Based Configuration

# Directory structure
cde-templates/
├── .git/
├── python-dev/
│   ├── main.tf
│   ├── devcontainer.json
│   └── README.md
├── java-spring/
│   ├── main.tf
│   └── docker-compose.yml
└── data-science/
    ├── main.tf
    └── requirements.txt

# Push templates to multi-region repos
git remote add origin-us \
  [email protected]:company/cde-templates.git
git remote add origin-eu \
  [email protected]:platform/cde-templates.git

git push origin-us main
git push origin-eu main

Container Image Replication

# AWS ECR - Enable cross-region replication
aws ecr put-replication-configuration \
  --replication-configuration '{
    "rules": [{
      "destinations": [{
        "region": "eu-west-1",
        "registryId": "123456789012"
      }]
    }]
  }'

# Azure ACR - Geo-replication
az acr replication create \
  --registry cderegistry \
  --location westeurope

# Manual image copy
docker pull us.gcr.io/company/workspace:latest
docker tag us.gcr.io/company/workspace:latest \
  eu.gcr.io/company/workspace:latest
docker push eu.gcr.io/company/workspace:latest

Sample Scenarios

Exercise Format

Control Plane Failure

# Kill control plane pods
kubectl delete pod -l app=coder \
  -n coder-system

# Verify:
# - Kubernetes restarts pods
# - Health checks detect failure
# - Alerting triggers
# - Workspaces unaffected

Database Failure Simulation

# Simulate DB connection loss
kubectl scale deployment coder \
  --replicas=0 -n coder-system

# Or use network policies
kubectl apply -f - <