nextxch highly available across multiple providers

; Zone example.com (TTL in seconds). IPs are documentation ranges: replace them.
; Site A = Hetzner 192.0.2.10, site B = netcup 198.51.100.20, site C (medium setup) = 203.0.113.30

; Inbound SMTP: equal preference, senders MUST randomize (RFC 5321 section 5.1).
example.com.            3600 IN MX 10 mx1.example.com.
example.com.            3600 IN MX 10 mx2.example.com.

; Static per-site names: never touched by the failover job (stable FCrDNS for PTR).
mx1.example.com.        3600 IN A     192.0.2.10
mx2.example.com.        3600 IN A     198.51.100.20

; Client name (IMAP, submission, HTTPS, webmail, ActiveSync, autodiscover):
; short TTL, kept up to date by the health-check job. Start with all healthy nodes.
mail.example.com.         60 IN A     192.0.2.10
mail.example.com.         60 IN A     198.51.100.20

; SPF: every outbound IP of every site. ip4/ip6 do not count against the 10-lookup limit (RFC 7208 4.6.4).
example.com.            3600 IN TXT   "v=spf1 ip4:192.0.2.10 ip4:198.51.100.20 ip4:203.0.113.30 -all"

; DKIM: nextxch keeps the keys in the cluster database, so one record set serves all nodes.
; Publish the selectors shown under /admin/ (Domains > DKIM).

; MTA-STS: the policy must list every MX host (see the note on gaps).
_mta-sts.example.com.   3600 IN TXT   "v=STSv1; id=20261102"

; DANE (optional, needs a DNSSEC-signed zone): one TLSA record per MX host.
; nextxch reuses the leaf key on renewal, so "3 1 1" stays stable.
_25._tcp.mx1.example.com. 3600 IN TLSA 3 1 1 <sha256-of-spki>
_25._tcp.mx2.example.com. 3600 IN TLSA 3 1 1 <sha256-of-spki>

; PTR (set in the provider console, not in this zone):
;   192.0.2.10    -> mx1.example.com   (Hetzner Console / Robot)
;   198.51.100.20 -> mx2.example.com   (netcup SCP)

# appsettings.Production.json per node (or admin: Hostname, IPv6 & Ports > EHLO name per node)
# Site A (Hetzner):
{ "Smtp": { "ServerHostName": "mail.example.com", "HeloHostname": "mx1.example.com" } }
# Site B (netcup):
{ "Smtp": { "ServerHostName": "mail.example.com", "HeloHostname": "mx2.example.com" } }
# PTR 192.0.2.10 -> mx1.example.com, PTR 198.51.100.20 -> mx2.example.com

# Served by nextxch itself at https://mta-sts.example.com/.well-known/mta-sts.txt
# (admin: Domains > domain > Transport > MTA-STS). Empty MX list = automatic:
# the domain's MX records from DNS + server host name + EHLO names of all nodes.
# Wildcards like *.example.com are allowed. Every change changes the policy id;
# the DNS advisor shows the matching _mta-sts TXT record.
version: STSv1
mode: enforce
mx: mx1.example.com
mx: mx2.example.com
max_age: 604800

#cloud-config
# Hetzner Cloud / netcup: paste as user data when creating the VM.
users:
  - name: deploy
    groups: [sudo, docker]
    shell: /bin/bash
    sudo: "ALL=(ALL) NOPASSWD:ALL"
    ssh_authorized_keys:
      - ssh-ed25519 AAAA... ci-deploy
package_update: true
packages: [wireguard, curl, jq, docker.io, docker-compose-v2, unattended-upgrades]
runcmd:
  - [ systemctl, enable, --now, docker ]
  - [ sed, -i, "s/^#\\?PasswordAuthentication.*/PasswordAuthentication no/", /etc/ssh/sshd_config ]
  - [ systemctl, restart, ssh ]

# /etc/wireguard/wg0.conf on site A (Hetzner). Same pattern on B and C.
# Full mesh: every site has one [Peer] block per other site.
[Interface]
Address    = 10.88.0.1/24
ListenPort = 51820
PrivateKey = <private-key-of-site-A>   # wg genkey, never commit it

[Peer]  # site B (netcup)
PublicKey           = <public-key-of-site-B>
Endpoint            = 198.51.100.20:51820
AllowedIPs          = 10.88.0.2/32
PersistentKeepalive = 25

[Peer]  # site C (witness or third site)
PublicKey           = <public-key-of-site-C>
Endpoint            = 203.0.113.30:51820
AllowedIPs          = 10.88.0.3/32
PersistentKeepalive = 25

# systemctl enable --now wg-quick@wg0
# Firewall: UDP 51820 open only for the two peer IPs.

# /etc/default/etcd on site A (Debian package etcd-server). Sites B and C analogue.
# Three members = quorum of two: any single site may fail.
ETCD_NAME="site-a"
ETCD_DATA_DIR="/var/lib/etcd/default"
ETCD_LISTEN_PEER_URLS="http://10.88.0.1:2380"
ETCD_LISTEN_CLIENT_URLS="http://10.88.0.1:2379,http://127.0.0.1:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.88.0.1:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://10.88.0.1:2379"
ETCD_INITIAL_CLUSTER="site-a=http://10.88.0.1:2380,site-b=http://10.88.0.2:2380,site-c=http://10.88.0.3:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="nextxch-ha"
# WAN tuning (etcd tuning guide: election timeout at least 10x the round trip).
# Example for up to ~25 ms RTT between providers:
ETCD_HEARTBEAT_INTERVAL="250"
ETCD_ELECTION_TIMEOUT="2500"
# Traffic stays inside WireGuard. For production add TLS (ETCD_CERT_FILE etc.).

# /etc/patroni/config.yml on site A (Debian/PGDG package "patroni").
# Site B: name/addresses analogue. The witness C runs etcd only, no PostgreSQL.
scope: nextxch
name: pg-site-a

restapi:
  listen: 10.88.0.1:8008
  connect_address: 10.88.0.1:8008

etcd3:
  hosts: 10.88.0.1:2379,10.88.0.2:2379,10.88.0.3:2379

bootstrap:
  dcs:
    ttl: 30                 # defaults; rule: loop_wait + 2 * retry_timeout <= ttl
    loop_wait: 10
    retry_timeout: 10
    maximum_lag_on_failover: 1048576
    synchronous_mode: on    # a 250 OK to a sender is only given after the commit reached a 2nd site
    synchronous_mode_strict: false   # if the replica site is gone, keep accepting mail (falls back to async)
    failsafe_mode: true
    postgresql:
      use_pg_rewind: true
      parameters:
        wal_level: replica
        hot_standby: "on"
        max_wal_senders: 10
        max_replication_slots: 10
        wal_log_hints: "on"
        archive_mode: "on"
        archive_command: "pgbackrest --stanza=nextxch archive-push %p"
  initdb: [ {encoding: UTF8}, data-checksums ]

postgresql:
  listen: 10.88.0.1:5432
  connect_address: 10.88.0.1:5432
  data_dir: /var/lib/postgresql/18/main
  bin_dir: /usr/lib/postgresql/18/bin
  authentication:
    replication: { username: replicator, password: "${PATRONI_REPLICATION_PASSWORD}" }
    superuser:   { username: postgres,   password: "${PATRONI_SUPERUSER_PASSWORD}" }
  pg_hba:
    - host replication replicator 10.88.0.0/24 scram-sha-256
    - host nextxch     nextxch    10.88.0.0/24 scram-sha-256

# /etc/garage.toml, identical on all three sites except rpc_public_addr.
metadata_dir = "/var/lib/garage/meta"
data_dir     = "/var/lib/garage/data"
db_engine    = "lmdb"

replication_factor = 3          # one copy per zone (site); writes need 2 of 3
consistency_mode   = "consistent"

rpc_bind_addr   = "10.88.0.1:3901"
rpc_public_addr = "10.88.0.1:3901"
rpc_secret      = "<openssl rand -hex 32, same on all nodes>"

[s3_api]
s3_region     = "garage"
api_bind_addr = "127.0.0.1:3900"   # each nextxch node talks to its local Garage

# After the first start (node ids from `garage node id`):
#   garage layout assign <id-a> -z hetzner -c 500G
#   garage layout assign <id-b> -z netcup  -c 500G
#   garage layout assign <id-c> -z witness -c 500G
#   garage layout show && garage layout apply --version 1
#   garage bucket create nextxch
#   garage key create nextxch-app
#   garage bucket allow --read --write --owner nextxch --key nextxch-app

# docker-compose.yml on site A and site B (nextxch only; PostgreSQL/Patroni, etcd,
# Garage and WireGuard run as system services on the same VM).
# Secrets come from .env (never committed): NEXTXCH_LICENSE, PG_PASSWORD, S3_KEY, S3_SECRET
name: nextxch

services:
  nextxch:
    image: nextxch/server:${NEXTXCH_TAG}
    restart: unless-stopped
    network_mode: host            # real client IPs for SPF/DMARC, direct WireGuard reach
    environment:
      ASPNETCORE_URLS: "http://*:80;https://*:443"
      License__Token: "${NEXTXCH_LICENSE}"
      Cluster__Enabled: "true"
      Cluster__NodeId: "${SITE_NAME}"          # hetzner-a / netcup-b
      Storage__Provider: "Postgres"
      # Npgsql multi-host: always connects to the current Patroni primary
      Storage__PostgresConnectionString: "Host=10.88.0.1,10.88.0.2;Port=5432;Database=nextxch;Username=nextxch;Password=${PG_PASSWORD};Target Session Attributes=primary;Host Recheck Seconds=5"
      Storage__BlobBackend: "S3"
      Storage__Compression: "Zstd"
      Storage__S3__EndpointUrl: "http://127.0.0.1:3900"
      Storage__S3__Region: "garage"
      Storage__S3__BucketName: "nextxch"
      Storage__S3__ForcePathStyle: "true"
      Storage__S3__AccessKey: "${S3_KEY}"
      Storage__S3__SecretKey: "${S3_SECRET}"
      Smtp__ServerHostName: "mail.example.com"
      Imap__ServerHostName: "mail.example.com"
      ManageSieve__ServerHostName: "mail.example.com"
      Acme__Enabled: "true"
      Acme__ContactEmail: "postmaster@example.com"
    volumes:
      - nextxch-data:/app/data
    healthcheck:
      test: ["CMD", "curl", "-fsS", "http://127.0.0.1/health/ready"]
      interval: 15s
      timeout: 5s
      retries: 3

volumes:
  nextxch-data:

# Admin: Domains > DNS failover (cluster). Stored cluster-wide; the token is encrypted
# with the server keyring and never shown again. Equivalent settings:
Provider:          Hetzner DNS (Cloud API, api.hetzner.cloud)   | Cloudflare DNS
Zone:              example.com
Names:             mail.example.com            # A/AAAA follow the healthy nodes
TTL:               60                          # resolvers keep old answers this long
Interval:          30 s
Failures (N):      3                           # 3 x 30 s before a node leaves DNS
Successes (M):     5                           # 5 x 30 s before it returns
Checks:            SMTP 25 (220 banner), HTTPS /health
Nodes:             fsn1, 192.0.2.10, 2001:db8::10
                   nbg1, 198.51.100.20
# Alert: rule "DNS failover: node taken out" (Alerts) fires when a node is out of DNS.

#!/usr/bin/env bash
# /usr/local/bin/nextxch-dns-failover.sh
# ALTERNATIVE to the built-in agent (admin: Domains > DNS failover), e.g. for DNS
# providers without a built-in connector (deSEC) or when etcd decides anyway.
# Keeps the A records of mail.example.com equal to the set of healthy nodes.
# Runs on every site under an etcd lock, so only one copy acts, and only on the
# side that still has etcd quorum (a cut-off minority site can never change DNS).
#   ExecStart=etcdctl --endpoints=http://127.0.0.1:2379 lock nextxch-dns /usr/local/bin/nextxch-dns-failover.sh
# Needs: curl, jq. Secrets via systemd EnvironmentFile (CF_API_TOKEN, CF_ZONE_ID).
set -euo pipefail

NAME="mail.example.com"
TTL=60
NODES=("192.0.2.10" "198.51.100.20")     # public IPs of the nextxch nodes
FAILS_BEFORE_REMOVE=3                    # 3 x 20 s = 1 minute
STATE=/run/nextxch-failover; mkdir -p "$STATE"
API="https://api.cloudflare.com/client/v4/zones/${CF_ZONE_ID}/dns_records"
AUTH=(-H "Authorization: Bearer ${CF_API_TOKEN}" -H "Content-Type: application/json")

healthy() {   # HTTPS readiness through the public IP, with SNI for the real name
  curl -fsS --max-time 5 --resolve "${NAME}:443:$1" "https://${NAME}/health/ready" \
    | jq -e '.status == "ready"' >/dev/null
}

while true; do
  want=()
  for ip in "${NODES[@]}"; do
    f="$STATE/$ip"; n=$(cat "$f" 2>/dev/null || echo 0)
    if healthy "$ip"; then echo 0 > "$f"; want+=("$ip")
    else n=$((n+1)); echo "$n" > "$f"; [ "$n" -lt "$FAILS_BEFORE_REMOVE" ] && want+=("$ip"); fi
  done
  # Never publish an empty set: if everything looks down, the checker is probably the problem.
  [ "${#want[@]}" -eq 0 ] && { sleep 20; continue; }

  have=$(curl -fsS "${AUTH[@]}" "${API}?type=A&name=${NAME}" | jq -r '.result[] | "\(.id) \(.content)"')
  for ip in "${want[@]}"; do   # add missing healthy IPs
    grep -q " ${ip}$" <<<"$have" || curl -fsS "${AUTH[@]}" -X POST "$API" \
      --data "{\"type\":\"A\",\"name\":\"${NAME}\",\"content\":\"${ip}\",\"ttl\":${TTL},\"proxied\":false}" >/dev/null
  done
  while read -r id ip; do      # remove IPs that are no longer healthy
    [ -z "${id:-}" ] && continue
    printf '%s\n' "${want[@]}" | grep -qx "$ip" || curl -fsS "${AUTH[@]}" -X DELETE "${API}/${id}" >/dev/null
  done <<<"$have"
  sleep 20
done

# /etc/systemd/system/nextxch-dns-failover.service (install on every site)
[Unit]
Description=nextxch DNS failover (runs only while holding the etcd lock)
After=network-online.target etcd.service wg-quick@wg0.service
Wants=network-online.target

[Service]
EnvironmentFile=/etc/nextxch/dns-failover.env
ExecStart=/usr/bin/etcdctl --endpoints=http://127.0.0.1:2379 lock nextxch-dns /usr/local/bin/nextxch-dns-failover.sh
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target

# inventory/ha.yml: three sites, one VM (or Proxmox guest) per site
all:
  vars:
    ansible_user: deploy
    wg_network: 10.88.0.0/24
    pg_major: 18
  children:
    nextxch_sites:
      hosts:
        hetzner-a: { ansible_host: 192.0.2.10,    wg_ip: 10.88.0.1, garage_zone: hetzner }
        netcup-b:  { ansible_host: 198.51.100.20, wg_ip: 10.88.0.2, garage_zone: netcup }
        third-c:   { ansible_host: 203.0.113.30,  wg_ip: 10.88.0.3, garage_zone: third }

# site.yml: idempotent base for every site (same playbook for Hetzner, netcup, Strato ...)
- hosts: nextxch_sites
  become: true
  tasks:
    - name: Base packages (Debian 13)
      ansible.builtin.apt:
        name: [wireguard, etcd-server, etcd-client, patroni, pgbackrest, docker.io, docker-compose-v2, jq, curl]
        update_cache: true
    # PostgreSQL 18 itself comes from the PGDG apt repository (apt.postgresql.org),
    # Garage as the static binary or the dxflrs/garage image.

    - name: WireGuard config
      ansible.builtin.template: { src: wg0.conf.j2, dest: /etc/wireguard/wg0.conf, mode: "0600" }
      notify: restart wg

    - name: etcd, Patroni and Garage config
      ansible.builtin.template: { src: "{{ item.src }}", dest: "{{ item.dest }}", mode: "0640" }
      loop:
        - { src: etcd.env.j2,     dest: /etc/default/etcd }
        - { src: patroni.yml.j2,  dest: /etc/patroni/config.yml }
        - { src: garage.toml.j2,  dest: /etc/garage.toml }

    - name: nextxch compose file
      ansible.builtin.template: { src: docker-compose.yml.j2, dest: /opt/nextxch/docker-compose.yml }

    - name: Start nextxch
      ansible.builtin.command: docker compose up -d
      args: { chdir: /opt/nextxch }
      changed_when: false

  handlers:
    - name: restart wg
      ansible.builtin.service: { name: wg-quick@wg0, state: restarted }

# values-hetzner.yaml (one file per provider cluster; netcup/strato analogue)
# Helm chart: deploy/helm/nextxch in the nextxch repository.
replicaCount: 3
mail:
  hostname: mail.example.com
  defaultDomain: example.com
postgres:
  # Patroni data layer outside Kubernetes, reached through WireGuard / KubeSpan
  connectionString: "Host=10.88.0.1,10.88.0.2,10.88.0.3;Database=nextxch;Username=nextxch;Password=<from secret>;Target Session Attributes=primary"
s3:
  endpointUrl: "http://garage.nextxch-data.svc:3900"   # Garage zone of this provider
  bucketName: "nextxch"

# .github/workflows/deploy.yml: rolling update of the compose sites, one site at a time.
name: deploy-nextxch
on:
  push:
    branches: [main]
    paths: ["sites/**", "compose/**"]
  workflow_dispatch:

concurrency: { group: deploy-nextxch, cancel-in-progress: false }

jobs:
  deploy:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 1            # never update two sites at once
      matrix:
        site: [hetzner-a, netcup-b]
    environment: ${{ matrix.site }}
    steps:
      - uses: actions/checkout@v4
      - name: Write SSH key
        run: |
          install -m 700 -d ~/.ssh
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/id_ed25519 && chmod 600 ~/.ssh/id_ed25519
          echo "${{ vars.HOST_KEY }}" >> ~/.ssh/known_hosts
      - name: Upload compose file and roll the container
        env:
          HOST: ${{ vars.SSH_HOST }}
        run: |
          scp compose/docker-compose.yml "deploy@${HOST}:/opt/nextxch/docker-compose.yml"
          scp "sites/${{ matrix.site }}.env.tag" "deploy@${HOST}:/opt/nextxch/tag.env"
          ssh "deploy@${HOST}" 'cd /opt/nextxch && docker compose --env-file .env --env-file tag.env pull && docker compose --env-file .env --env-file tag.env up -d'
      - name: Wait until the node is ready again
        env:
          HOST: ${{ vars.SSH_HOST }}
        run: |
          for i in $(seq 1 30); do
            ssh "deploy@${HOST}" 'curl -fsS http://127.0.0.1/health/ready' && exit 0
            sleep 10
          done
          echo "node did not become ready" >&2; exit 1

# .gitlab-ci.yml: the same rolling update with GitLab CI
stages: [validate, deploy-a, deploy-b]

variables:
  NEXTXCH_TAG: "latest"     # pin a released version in production

validate:
  stage: validate
  image: docker:27
  script:
    - docker compose -f compose/docker-compose.yml config -q

.deploy: &deploy
  image: alpine:3.22
  before_script:
    - apk add --no-cache openssh-client curl
    - install -m 700 -d ~/.ssh && echo "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519 && chmod 600 ~/.ssh/id_ed25519
    - echo "$HOST_KEY" >> ~/.ssh/known_hosts
  script:
    - scp compose/docker-compose.yml "deploy@$SSH_HOST:/opt/nextxch/"
    - ssh "deploy@$SSH_HOST" "cd /opt/nextxch && NEXTXCH_TAG=$NEXTXCH_TAG docker compose pull && NEXTXCH_TAG=$NEXTXCH_TAG docker compose up -d"
    - for i in $(seq 1 30); do ssh "deploy@$SSH_HOST" 'curl -fsS http://127.0.0.1/health/ready' && exit 0; sleep 10; done; exit 1
  resource_group: production   # serializes deployments
  rules: [ { if: '$CI_COMMIT_BRANCH == "main"' } ]

deploy-hetzner-a:
  <<: *deploy
  stage: deploy-a
  environment: { name: hetzner-a }

deploy-netcup-b:
  <<: *deploy
  stage: deploy-b
  environment: { name: netcup-b }

# .github/workflows/deploy-k8s.yml: one Kubernetes cluster per provider, rolled one after another.
name: deploy-nextxch-k8s
on:
  push: { branches: [main], paths: ["k8s/**"] }
  workflow_dispatch:

concurrency: { group: deploy-k8s, cancel-in-progress: false }

jobs:
  helm:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 1
      matrix:
        cluster: [hetzner, netcup, strato]
    environment: k8s-${{ matrix.cluster }}
    steps:
      - uses: actions/checkout@v4
      - uses: azure/setup-helm@v4
      - name: Kubeconfig
        run: |
          mkdir -p ~/.kube
          echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > ~/.kube/config
          chmod 600 ~/.kube/config
      - name: Helm upgrade (waits for readiness probes on /health/ready)
        run: |
          helm upgrade --install nextxch ./deploy/helm/nextxch \
            --namespace nextxch --create-namespace \
            -f k8s/values-common.yaml -f k8s/values-${{ matrix.cluster }}.yaml \
            --set image.tag=${{ vars.NEXTXCH_TAG }} \
            --wait --timeout 10m --atomic

# /etc/pgbackrest/pgbackrest.conf on every PostgreSQL site.
# Repo 1: Hetzner Storage Box over SFTP (pgBackRest 2.46+), repo 2: S3 at another provider.
[global]
repo1-type=sftp
repo1-path=/pgbackrest
repo1-sftp-host=uXXXXXX.your-storagebox.de
repo1-sftp-host-port=23
repo1-sftp-host-user=uXXXXXX
repo1-sftp-private-key-file=/var/lib/postgresql/.ssh/id_ed25519
repo1-sftp-host-key-hash-type=sha256
repo1-retention-full=2
repo1-cipher-type=aes-256-cbc
repo1-cipher-pass=<long random secret, stored outside the repo>

repo2-type=s3
repo2-path=/pgbackrest
repo2-s3-endpoint=<s3 endpoint of the second provider>
repo2-s3-bucket=nextxch-pg-backup
repo2-s3-region=<region>
repo2-s3-uri-style=path
repo2-retention-full=4
repo2-cipher-type=aes-256-cbc
repo2-cipher-pass=<second secret>

archive-async=y
process-max=2
start-fast=y

[nextxch]
pg1-path=/var/lib/postgresql/18/main
pg1-port=5432

# Weekly full, daily differential (cron on every node; pgBackRest runs only where it can,
# use --repo and the "backup-standby" option to take backups from a replica).
#   0 2 * * 0  pgbackrest --stanza=nextxch --type=full backup
#   0 2 * * 1-6 pgbackrest --stanza=nextxch --type=diff backup

# Blob copy (all .eml objects) from Garage to an independent target, every hour.
# ~/.config/rclone/rclone.conf (keys are placeholders)
[garage]
type = s3
provider = Other
endpoint = http://127.0.0.1:3900
region = garage
access_key_id = <garage key>
secret_access_key = <garage secret>

[archive]
type = s3
provider = Other
endpoint = <object storage endpoint>
access_key_id = <key>
secret_access_key = <secret>

# Content-addressed (SHA-256 names, never modified): "copy" is enough and never deletes.
#   rclone copy garage:nextxch archive:nextxch-archive --immutable --transfers 8 --checksum
# For a GoBD copy create the target bucket WITH Object Lock and a default retention
# (COMPLIANCE, e.g. 10 years) so every copied object is locked automatically.