didbot_tls/

fleet.rs

//! One certificate pair per zone, and the SNI dispatch that picks between
//! them.
//!
//! # Why a deployment cannot have one certificate
//!
//! A wildcard matches exactly one label. `*.pds.did.bot` covers
//! `claudes.pds.did.bot` and does not cover `agent.claudes.pds.did.bot`, so
//! a deployment configured with `claudes.pds.did.bot` and
//! `vllm.pds.did.bot` needs a `<zone>` + `*.<zone>` pair for each of them —
//! two ACME orders, two certificates, two renewal clocks.
//! [`crate::manager::CertificateManager`] is one zone's worth of that;
//! [`CertificateFleet`] is the deployment's.
//!
//! # One `ServerConfig` regardless
//!
//! The listener still binds once and the process still builds one
//! `rustls::ServerConfig`. [`ZoneResolver`] sits where a single zone's
//! [`crate::resolver::SwappableCert`] would, holds one of those per zone,
//! and picks between them on the SNI name each `ClientHello` carries. Each
//! zone's renewal swaps its own certificate under the same
//! [`arc_swap::ArcSwap`] a single-zone deployment uses, so hot reload is
//! unchanged: no listener is rebound, and no connection past its handshake
//! notices.
//!
//! # What a failed renewal does here
//!
//! Exactly what it does for one zone, per zone and no more widely.
//! [`crate::renew`] holds the whole policy: renewal starts 30 days before
//! expiry, retries hourly, keeps serving the existing certificate on
//! failure, warns after three consecutive failures and escalates to
//! [`AlertLevel::Critical`] inside three days of expiry whether or not
//! anything has failed yet. A zone whose renewal is failing does not stop
//! the other zones' renewals or their traffic; [`CertificateFleet::alert_level`]
//! reports the worst level any zone is at, so a deployment-wide health check
//! reads one value, and [`CertificateFleet::zone_alert_levels`] says which
//! zone it came from.

use std::sync::Arc;

use rustls::server::{ClientHello, ResolvesServerCert};
use rustls::sign::CertifiedKey;

use crate::directory::Environment;
use crate::dns::AcmeDnsChallenge;
use crate::manager::{BootstrapError, CertificateManager};
use crate::renew::{AlertLevel, RenewalPolicy};
use crate::resolver::SwappableCert;

/// Why a fleet could not be assembled.
#[derive(Debug, thiserror::Error)]
pub enum FleetError {
    /// No zones were offered at all.
    #[error("a certificate fleet needs at least one zone")]
    NoZones,
    /// The same zone hostname was configured twice. Two managers renewing
    /// one zone would place two ACME orders for the same names and race each
    /// other's storage, so this is refused rather than deduplicated.
    #[error("zone {0:?} is configured more than once")]
    DuplicateZone(String),
    /// One zone's certificate could not be loaded or obtained. Named, because
    /// "the certificate failed" is not actionable in a deployment holding
    /// several.
    #[error("zone {zone:?}: {source}")]
    Zone {
        /// The zone that failed.
        zone: String,
        /// What went wrong for it.
        #[source]
        source: Box<BootstrapError>,
    },
}

/// Whether a certificate for `zone` — covering `zone` and `*.zone` — is the
/// right certificate to present for `host`.
///
/// The one-label rule, stated once: the apex itself, or exactly one label
/// under it. `agent.claudes.pds.did.bot` is *not* covered by a certificate
/// for `pds.did.bot`, which is the whole reason a deployment needs a
/// certificate per zone rather than one at the root.
pub fn zone_covers(zone: &str, host: &str) -> bool {
    let zone = zone.trim_end_matches('.');
    let host = host.trim_end_matches('.');
    if host.eq_ignore_ascii_case(zone) {
        return true;
    }
    match host.len().checked_sub(zone.len() + 1) {
        Some(prefix_len) if prefix_len > 0 => {
            let (prefix, rest) = host.split_at(prefix_len);
            rest.eq_ignore_ascii_case(&format!(".{zone}")) && !prefix.contains('.')
        }
        _ => false,
    }
}

/// The process's one `rustls` certificate resolver when several zones are
/// served: one [`SwappableCert`] per zone, chosen by SNI.
///
/// # No SNI, and unknown SNI
///
/// A `ClientHello` with no server name — an IP-address connection, or an
/// ancient client — gets the *primary* zone's certificate: the first zone
/// configured, which is also the zone whose hostname the server answers for
/// itself. Handing back something is right here; the handshake then fails or
/// succeeds on the client's own name check, which is where that decision
/// belongs.
///
/// A `ClientHello` naming a host no configured zone covers gets `None`,
/// which rustls turns into an `unrecognized_name` alert. That is the honest
/// answer: presenting a certificate for a name we do not serve would fail
/// the client's verification anyway, and more confusingly.
#[derive(Debug)]
pub struct ZoneResolver {
    /// Zone hostname and its live certificate, sorted deepest zone first so
    /// a nested pair (`pds.did.bot` and `claudes.pds.did.bot`) resolves to
    /// the more specific one — matching `didbot_identity::ZoneRegistry`'s
    /// own "most specific wins" rule.
    zones: Vec<(String, Arc<SwappableCert>)>,
    /// The first zone as *configured*, which sorting by depth moves. Held
    /// separately rather than by index for that reason.
    primary: Arc<SwappableCert>,
}

impl ZoneResolver {
    /// Builds a resolver over `zones`, refusing an empty or duplicated set.
    ///
    /// The first entry is the primary: the certificate a `ClientHello` with
    /// no SNI gets.
    pub fn new(zones: Vec<(String, Arc<SwappableCert>)>) -> Result<Arc<Self>, FleetError> {
        if zones.is_empty() {
            return Err(FleetError::NoZones);
        }
        for i in 0..zones.len() {
            for j in (i + 1)..zones.len() {
                if zones[i].0.eq_ignore_ascii_case(&zones[j].0) {
                    return Err(FleetError::DuplicateZone(zones[i].0.clone()));
                }
            }
        }
        let primary = zones[0].1.clone();
        let mut zones = zones;
        zones.sort_by_key(|(zone, _)| std::cmp::Reverse(zone.matches('.').count()));
        Ok(Arc::new(Self { zones, primary }))
    }

    /// The zones this resolver can answer for, deepest first.
    pub fn zones(&self) -> Vec<&str> {
        self.zones.iter().map(|(zone, _)| zone.as_str()).collect()
    }

    /// The certificate this resolver would present for `host`, if any.
    ///
    /// Exposed so a test — or an operator-facing check — can ask the
    /// question without building a `ClientHello`.
    pub fn certificate_for(&self, host: &str) -> Option<Arc<CertifiedKey>> {
        self.zones
            .iter()
            .find(|(zone, _)| zone_covers(zone, host))
            .map(|(_, cert)| cert.current())
    }
}

impl ResolvesServerCert for ZoneResolver {
    fn resolve(&self, client_hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
        match client_hello.server_name() {
            Some(host) => {
                let found = self.certificate_for(host);
                if found.is_none() {
                    tracing::debug!(
                        sni = host,
                        zones = ?self.zones(),
                        "no configured zone covers this sni name; refusing the handshake"
                    );
                }
                found
            }
            None => Some(self.primary_cert()),
        }
    }
}

impl ZoneResolver {
    /// The certificate a `ClientHello` with no SNI name gets: the first zone
    /// configured.
    pub fn primary_cert(&self) -> Arc<CertifiedKey> {
        self.primary.current()
    }
}

/// Every configured zone's certificate, renewed independently, served
/// through one `rustls::ServerConfig`.
pub struct CertificateFleet<D: AcmeDnsChallenge> {
    managers: Vec<Arc<CertificateManager<D>>>,
    resolver: Arc<ZoneResolver>,
}

impl<D: AcmeDnsChallenge + 'static> CertificateFleet<D> {
    /// Loads or obtains a `<zone>` + `*.<zone>` certificate for every entry
    /// in `zones`, each through its own DNS-01 backend.
    ///
    /// `zones` pairs a zone hostname with the backend that can write
    /// `_acme-challenge.<zone>` for it — in production one `Route53Dns` per
    /// hosted zone, since one credential is scoped to one zone. The first
    /// entry is the primary (see [`ZoneResolver`]).
    ///
    /// Fails if *any* zone fails: a server that came up serving three of its
    /// four configured zones would be a deployment silently missing a
    /// quarter of its agents, discovered by whoever tried to use them.
    pub async fn bootstrap(
        data_dir: &std::path::Path,
        zones: Vec<(String, Arc<D>)>,
        environment: Environment,
        contact: &[&str],
        policy: RenewalPolicy,
    ) -> Result<Arc<Self>, FleetError> {
        if zones.is_empty() {
            return Err(FleetError::NoZones);
        }
        for i in 0..zones.len() {
            for j in (i + 1)..zones.len() {
                if zones[i].0.eq_ignore_ascii_case(&zones[j].0) {
                    return Err(FleetError::DuplicateZone(zones[i].0.clone()));
                }
            }
        }

        let mut managers = Vec::with_capacity(zones.len());
        // Sequential, not concurrent: the first zone through here is the one
        // that creates the shared ACME account, and two registrations racing
        // for one account is a needless way to meet a CA's rate limit.
        for (zone, dns) in zones {
            let manager =
                CertificateManager::bootstrap(data_dir, &zone, environment, contact, policy, dns)
                    .await
                    .map_err(|source| FleetError::Zone {
                        zone: zone.clone(),
                        source: Box::new(source),
                    })?;
            managers.push(manager);
        }

        Self::from_managers(managers)
    }

    /// Assembles a fleet from managers built elsewhere. The escape hatch a
    /// test uses, and the seam that keeps [`Self::bootstrap`] from being the
    /// only way in.
    pub fn from_managers(
        managers: Vec<Arc<CertificateManager<D>>>,
    ) -> Result<Arc<Self>, FleetError> {
        let resolver = ZoneResolver::new(
            managers
                .iter()
                .map(|manager| (manager.zone().to_owned(), manager.resolver()))
                .collect(),
        )?;
        tracing::info!(zones = ?resolver.zones(), "tls certificates ready, one pair per zone");
        Ok(Arc::new(Self { managers, resolver }))
    }

    /// The SNI-dispatching resolver behind [`Self::server_config`].
    pub fn resolver(&self) -> Arc<ZoneResolver> {
        self.resolver.clone()
    }

    /// The one `rustls::ServerConfig` this process should build, whatever
    /// number of zones it holds. Renewal swaps a certificate underneath it;
    /// nothing rebuilds this or rebinds the listener.
    pub fn server_config(&self) -> Arc<rustls::ServerConfig> {
        let mut config = rustls::ServerConfig::builder()
            .with_no_client_auth()
            .with_cert_resolver(self.resolver.clone());
        config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
        Arc::new(config)
    }

    /// Every zone's manager, in configuration order.
    pub fn managers(&self) -> &[Arc<CertificateManager<D>>] {
        &self.managers
    }

    /// The worst alert level any zone is currently at.
    pub async fn alert_level(&self) -> AlertLevel {
        let mut worst = AlertLevel::Ok;
        for manager in &self.managers {
            worst = worst.max(manager.alert_level().await);
        }
        worst
    }

    /// Each zone's alert level, so a failing renewal is attributable to the
    /// zone it is failing for rather than to the deployment as a whole.
    pub async fn zone_alert_levels(&self) -> Vec<(String, AlertLevel)> {
        let mut levels = Vec::with_capacity(self.managers.len());
        for manager in &self.managers {
            levels.push((manager.zone().to_owned(), manager.alert_level().await));
        }
        levels
    }

    /// Runs every zone's renewal loop, forever.
    ///
    /// One task per zone, not one loop over all of them: a zone whose ACME
    /// exchange is hanging must not hold up another zone's renewal behind
    /// it. Cancelling this future aborts all of them.
    pub async fn run_renewal_loops(self: Arc<Self>) {
        let mut tasks = Vec::with_capacity(self.managers.len());
        for manager in &self.managers {
            tasks.push(tokio::spawn(manager.clone().run_renewal_loop()));
        }
        let _guard = AbortOnDrop(tasks);
        std::future::pending::<()>().await;
    }
}

/// Aborts every renewal task when the loop future is dropped, so a
/// cancelled `run_renewal_loops` does not leave tasks renewing behind it.
struct AbortOnDrop(Vec<tokio::task::JoinHandle<()>>);

impl Drop for AbortOnDrop {
    fn drop(&mut self) {
        for task in &self.0 {
            task.abort();
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::cert::certified_key_from_pem;
    use crate::test_support::self_signed_pem;

    fn swappable(name: &str) -> Arc<SwappableCert> {
        let (cert, key) = self_signed_pem(name);
        let provider = rustls::crypto::ring::default_provider();
        SwappableCert::new(certified_key_from_pem(&cert, &key, &provider).unwrap())
    }

    fn resolver(zones: &[&str]) -> Arc<ZoneResolver> {
        ZoneResolver::new(
            zones
                .iter()
                .map(|zone| ((*zone).to_owned(), swappable(zone)))
                .collect(),
        )
        .unwrap()
    }

    #[test]
    fn a_zone_covers_its_apex_and_exactly_one_label_under_it() {
        assert!(zone_covers("pds.did.bot", "pds.did.bot"));
        assert!(zone_covers("pds.did.bot", "claudes.pds.did.bot"));
        assert!(!zone_covers("pds.did.bot", "agent.claudes.pds.did.bot"));
        assert!(!zone_covers("pds.did.bot", "did.bot"));
        assert!(!zone_covers("pds.did.bot", "notpds.did.bot"));
        assert!(!zone_covers("pds.did.bot", ".pds.did.bot"));
    }

    #[test]
    fn coverage_ignores_case_and_a_trailing_root_dot() {
        assert!(zone_covers("pds.did.bot", "Claudes.PDS.did.bot"));
        assert!(zone_covers("pds.did.bot", "claudes.pds.did.bot."));
    }

    #[test]
    fn each_zone_gets_its_own_certificate() {
        let resolver = resolver(&["claudes.pds.did.bot", "vllm.pds.did.bot"]);

        // An agent one label under each zone gets that zone's own
        // certificate, and the two are different certificates -- which is
        // the property a single deployment-wide pair cannot have.
        let a = resolver
            .certificate_for("agent.claudes.pds.did.bot")
            .unwrap();
        let b = resolver.certificate_for("agent.vllm.pds.did.bot").unwrap();
        assert_ne!(a.cert.first().unwrap(), b.cert.first().unwrap());
        assert_eq!(
            a.cert.first().unwrap(),
            resolver
                .certificate_for("claudes.pds.did.bot")
                .unwrap()
                .cert
                .first()
                .unwrap()
        );

        // Two labels under a zone is past what its wildcard covers, and
        // neither zone's certificate is offered for it.
        assert!(resolver
            .certificate_for("agent.sub.claudes.pds.did.bot")
            .is_none());
    }

    #[test]
    fn a_name_no_zone_covers_gets_nothing() {
        let resolver = resolver(&["claudes.pds.did.bot"]);
        assert!(resolver.certificate_for("elsewhere.example").is_none());
        assert!(resolver
            .certificate_for("deep.one.claudes.pds.did.bot")
            .is_none());
    }

    #[test]
    fn a_nested_zone_wins_over_its_parent() {
        // Configuration order deliberately puts the parent first: the
        // resolver must sort by depth, not trust the order it was given.
        let resolver = resolver(&["pds.did.bot", "claudes.pds.did.bot"]);
        let nested = resolver
            .certificate_for("agent.claudes.pds.did.bot")
            .unwrap();
        let own = resolver.certificate_for("claudes.pds.did.bot").unwrap();
        // `claudes.pds.did.bot` is covered by both zones; the deeper one's
        // certificate is the one that also covers what is under it, so it is
        // the one that must answer for both names.
        assert_eq!(nested.cert.first().unwrap(), own.cert.first().unwrap());
    }

    #[test]
    fn no_sni_gets_the_primary_zone() {
        let resolver = resolver(&["pds.did.bot", "claudes.pds.did.bot"]);
        let primary = resolver.certificate_for("pds.did.bot").unwrap();
        assert_eq!(
            resolver.primary_cert().cert.first().unwrap(),
            primary.cert.first().unwrap()
        );
    }

    #[test]
    fn a_renewal_swaps_only_its_own_zone() {
        let claudes = swappable("claudes.pds.did.bot");
        let vllm = swappable("vllm.pds.did.bot");
        let resolver = ZoneResolver::new(vec![
            ("claudes.pds.did.bot".to_owned(), claudes.clone()),
            ("vllm.pds.did.bot".to_owned(), vllm.clone()),
        ])
        .unwrap();

        let vllm_before = resolver.certificate_for("vllm.pds.did.bot").unwrap();
        let claudes_before = resolver.certificate_for("claudes.pds.did.bot").unwrap();

        let (cert, key) = self_signed_pem("claudes.pds.did.bot");
        let provider = rustls::crypto::ring::default_provider();
        claudes.swap(certified_key_from_pem(&cert, &key, &provider).unwrap());

        assert_ne!(
            claudes_before.cert.first().unwrap(),
            resolver
                .certificate_for("claudes.pds.did.bot")
                .unwrap()
                .cert
                .first()
                .unwrap()
        );
        assert_eq!(
            vllm_before.cert.first().unwrap(),
            resolver
                .certificate_for("vllm.pds.did.bot")
                .unwrap()
                .cert
                .first()
                .unwrap()
        );
    }

    #[test]
    fn duplicate_and_empty_zone_sets_are_refused() {
        assert!(matches!(
            ZoneResolver::new(vec![]).unwrap_err(),
            FleetError::NoZones
        ));
        let dup = ZoneResolver::new(vec![
            ("pds.did.bot".to_owned(), swappable("pds.did.bot")),
            ("PDS.did.bot".to_owned(), swappable("pds.did.bot")),
        ]);
        assert!(matches!(dup.unwrap_err(), FleetError::DuplicateZone(_)));
    }
}