didbot_tls/

manager.rs

//! Ties storage, the DNS-01 seam, the renewal clock and the swappable
//! certificate together into the one thing a server needs: a
//! [`rustls::ServerConfig`] that renews itself.
//!
//! # What this crate's tests verify, and what they still cannot
//!
//! `../tests/acme_conformance.rs` drives the full `instant_acme` exchange
//! this module orders through -- `new_order`, both authorizations, `dns-01`
//! challenges, `finalize`, `certificate` -- against an in-process fake ACME
//! directory (`../tests/support::FakeAcmeServer`), through
//! [`CertificateManager::bootstrap_with_account`]. That covers: the
//! apex+wildcard grouping actually reaching a CA-shaped server rather than
//! only the pure grouping function; a challenge that never validates; a
//! renewal that fails
//! repeatedly (still serving the old certificate, warning after three
//! failures, critical inside three days of expiry); a crash between
//! publishing a challenge and withdrawing it; and two zones ordering
//! concurrently without sharing a challenge name.
//!
//! What that still does not cover: a real CA (rate limits, real validation,
//! real certificate chains -- the fake signs the order's CSR key with a
//! throwaway issuer nothing trusts), real DNS-01 propagation over the
//! network, and [`ReqwestAcmeClient`] actually reaching a directory --
//! [`CertificateManager::bootstrap`] itself is real-network-only and has
//! no fake-directory path of its own, which is exactly why
//! [`CertificateManager::bootstrap_with_account`] exists as a seam. Also exercised, independent of any of the above, purely
//! as functions of a clock the tests supply:
//!
//! * [`crate::renew::RenewalTracker`] -- when a renewal is due and how loud
//!   a failing one gets.
//! * `crate::acme::group_challenges_by_name` -- the grouping that avoids
//!   publishing the apex's DNS-01 value and then overwriting it with the
//!   wildcard's, or vice versa.
//! * [`crate::resolver::SwappableCert`] -- that a certificate swap is
//!   invisible to a connection already past its handshake, and visible to
//!   the very next one. See `tests/rotation.rs`.
//! * [`crate::storage`] -- permissions and round-tripping on disk.

use std::io;
use std::sync::Arc;
use std::time::Duration;

use instant_acme::{Account, NewAccount};
use rustls::sign::CertifiedKey;
use time::OffsetDateTime;
use tokio::sync::Mutex;

use crate::acme::{self, IssueError};
use crate::cert::{certified_key_from_pem, leaf_expiry, CertError};
use crate::directory::Environment;
use crate::dns::{AcmeDnsChallenge, IssuancePolicy};
use crate::http_client::ReqwestAcmeClient;
use crate::renew::{AlertLevel, RenewalPolicy, RenewalTracker};
use crate::resolver::SwappableCert;
use crate::storage::{CertMeta, CertStore, StorageError};

/// How often the background loop checks whether renewal is due. Independent
/// of [`RenewalPolicy::retry_interval`]: this is the poll granularity, that
/// is the minimum gap between actual attempts.
pub const CHECK_INTERVAL: Duration = Duration::from_secs(3600);

/// Why the manager could not be built, or could not obtain a first
/// certificate.
#[derive(Debug, thiserror::Error)]
pub enum BootstrapError {
    /// The data directory could not be prepared, or a saved file did not
    /// parse.
    #[error("data directory: {0}")]
    Storage(#[from] StorageError),
    /// Creating or restoring the ACME account failed.
    #[error("acme: {0}")]
    Acme(#[from] instant_acme::Error),
    /// No certificate was on disk and the first issuance failed.
    #[error("issuing the first certificate: {0}")]
    Issue(#[from] IssueError),
    /// The certificate this crate just obtained or loaded from disk could
    /// not be turned into something `rustls` can serve.
    #[error("the certificate this crate just obtained or loaded: {0}")]
    Cert(#[from] CertError),
}

/// The PDS's own certificate: issuance, renewal and the live rustls state,
/// in one place.
pub struct CertificateManager<D: AcmeDnsChallenge> {
    store: CertStore,
    dns: Arc<D>,
    account: Account,
    environment: Environment,
    domains: Vec<String>,
    policy: RenewalPolicy,
    tracker: Mutex<RenewalTracker>,
    cert: Arc<SwappableCert>,
    provider: rustls::crypto::CryptoProvider,
}

/// The ACME account `directory_url` should be reached through: the one
/// saved in `store` if it was registered there, and a freshly registered one
/// otherwise, saved over it.
///
/// An account belongs to the directory that registered it. It can place
/// orders there and nowhere else, and the `CAA` record naming it authorises
/// issuance there and nowhere else — so a deployment that switches
/// directories (staging to production, the switch that makes its
/// certificates publicly trusted) needs an account at the new one before it
/// can order anything. Credentials that record no directory at all are
/// replaced too: one round trip to register an account is the cheaper half
/// of that guess.
///
/// Taking the [`instant_acme::HttpClient`] as an argument is the same seam
/// [`CertificateManager::bootstrap_with_account`] is:
/// [`CertificateManager::bootstrap`] passes a [`ReqwestAcmeClient`] against
/// a real Let's Encrypt directory, and a test passes a client answering as
/// an in-process fake one.
pub async fn load_or_register_account(
    store: &CertStore,
    directory_url: &str,
    contact: &[&str],
    http: Box<dyn instant_acme::HttpClient>,
) -> Result<Account, BootstrapError> {
    match store.account_credentials()? {
        Some(stored) if stored.issued_by(directory_url) => {
            tracing::info!(
                directory = directory_url,
                "restoring the saved acme account"
            );
            Ok(Account::builder_with_http(http)
                .from_credentials(stored.credentials)
                .await?)
        }
        stale => {
            if let Some(stale) = &stale {
                tracing::warn!(
                    saved = stale
                        .directory_url
                        .as_deref()
                        .unwrap_or("an unrecorded directory"),
                    configured = directory_url,
                    "the saved acme account belongs to another directory; registering a new \
                     account at the configured one"
                );
            }
            let (account, credentials) = Account::builder_with_http(http)
                .create(
                    &NewAccount {
                        contact,
                        terms_of_service_agreed: true,
                        only_return_existing: false,
                    },
                    directory_url.to_owned(),
                    None,
                )
                .await?;
            store.save_account_credentials(&credentials)?;
            Ok(account)
        }
    }
}

/// The certificate stored for `zone` if this run can serve it, with its
/// expiry and metadata. `None` means order a new one.
///
/// A certificate is only as good as the directory that signed it: a staging
/// leaf is served by exactly the same code as a production one and trusted by
/// nobody, so the environment a run is configured for decides whether what is
/// on disk can be served, not merely whether anything is there.
///
/// A set a crash left unusable is also `None`, so the next boot replaces it
/// rather than refusing to start: a file missing, a `meta.json` that does not
/// parse, or a certificate and key that do not load as a matching pair.
/// `CertStore::save_certificate` renames its three files one at a time, so a
/// crash between two renames leaves exactly such a set. Any other read error
/// is still returned, since ordering again would not fix it.
fn stored_certificate(
    store: &CertStore,
    zone: &str,
    environment: Environment,
    provider: &rustls::crypto::CryptoProvider,
) -> Result<Option<(CertifiedKey, OffsetDateTime, CertMeta)>, BootstrapError> {
    let (cert_pem, key_pem, meta) = match store.certificate() {
        Ok(Some(stored)) => stored,
        Ok(None) => {
            tracing::info!(zone, "no certificate on disk yet; requesting the first one");
            return Ok(None);
        }
        Err(StorageError::Io { path, source }) if source.kind() == io::ErrorKind::NotFound => {
            tracing::warn!(
                zone,
                missing = %path.display(),
                "the stored certificate is incomplete; ordering a new one"
            );
            return Ok(None);
        }
        Err(err @ StorageError::Parse { .. }) => {
            tracing::warn!(zone, %err, "the stored certificate metadata does not parse; ordering a new one");
            return Ok(None);
        }
        Err(err) => return Err(err.into()),
    };
    if meta.directory_url != environment.directory_url() {
        tracing::info!(
            zone,
            stored = %meta.directory_url,
            configured = environment.directory_url(),
            "the certificate on disk was issued by another directory; ordering one from the \
             configured directory"
        );
        return Ok(None);
    }
    let loaded = certified_key_from_pem(&cert_pem, &key_pem, provider)
        .and_then(|certified| Ok((leaf_expiry(&certified.cert)?, certified)));
    match loaded {
        Ok((expiry, certified)) => Ok(Some((certified, expiry, meta))),
        Err(err) => {
            tracing::warn!(zone, %err, "the stored certificate does not load; ordering a new one");
            Ok(None)
        }
    }
}

impl<D: AcmeDnsChallenge + 'static> CertificateManager<D> {
    /// Loads or obtains a certificate for `zone` and `*.zone`, from
    /// `data_dir/tls/zones/<zone>`, and returns a manager ready to serve and
    /// renew it.
    ///
    /// One manager covers exactly one zone. A deployment configured with
    /// several needs one of these each — [`crate::fleet::CertificateFleet`]
    /// is what builds and holds them.
    ///
    /// Installs `ring` as the process's default `rustls` crypto provider if
    /// none is installed yet; if one is already installed (for instance by
    /// another part of the process that also uses rustls) this is a no-op
    /// rather than an error -- a second install attempt is expected, not
    /// exceptional.
    pub async fn bootstrap(
        data_dir: &std::path::Path,
        zone: &str,
        environment: Environment,
        contact: &[&str],
        policy: RenewalPolicy,
        dns: Arc<D>,
    ) -> Result<Arc<Self>, BootstrapError> {
        tracing::info!("{}", environment.banner());

        // Per zone, not per deployment: `<data_dir>/tls/zones/<zone>` holds
        // this zone's certificate, `<data_dir>/tls` the one ACME account
        // every zone's orders are placed through. See `storage`'s own docs,
        // and `crate::fleet` for why one certificate can never cover two
        // zones.
        let store = CertStore::open_zone(data_dir, zone)?;

        let account = load_or_register_account(
            &store,
            environment.directory_url(),
            contact,
            Box::new(ReqwestAcmeClient::new()),
        )
        .await?;

        Self::finish_bootstrap(store, zone, environment, contact, policy, dns, account).await
    }

    /// Assembles a manager from an [`Account`] built elsewhere, skipping
    /// [`CertificateManager::bootstrap`]'s own credential-loading and its hardcoded
    /// [`ReqwestAcmeClient`] against `environment.directory_url()`.
    ///
    /// This is the seam a test drives against an in-process fake ACME
    /// directory: [`CertificateManager::bootstrap`] has no way to reach anything but a
    /// real Let's Encrypt endpoint, so nothing downstream of it could run
    /// without one. Production code that already holds an [`Account`] -- for
    /// instance one restored through a different credential path than
    /// [`crate::storage::CertStore`] -- can use this directly too.
    pub async fn bootstrap_with_account(
        data_dir: &std::path::Path,
        zone: &str,
        environment: Environment,
        contact: &[&str],
        policy: RenewalPolicy,
        dns: Arc<D>,
        account: Account,
    ) -> Result<Arc<Self>, BootstrapError> {
        let store = CertStore::open_zone(data_dir, zone)?;
        Self::finish_bootstrap(store, zone, environment, contact, policy, dns, account).await
    }

    /// Everything both bootstrap paths share once an [`Account`] is in hand:
    /// publish the zone's `CAA` record, load or issue the certificate, and
    /// assemble the manager.
    async fn finish_bootstrap(
        store: CertStore,
        zone: &str,
        environment: Environment,
        contact: &[&str],
        policy: RenewalPolicy,
        dns: Arc<D>,
        account: Account,
    ) -> Result<Arc<Self>, BootstrapError> {
        let _ = rustls::crypto::ring::default_provider().install_default();
        let provider = rustls::crypto::ring::default_provider();
        let domains = acme::zone_and_wildcard(zone);

        // Before any order, and on every start rather than only the first:
        // the record has to name the account this process actually holds, and
        // that is only knowable here, once the account is loaded or created.
        // A backend that already says this skips the write. This is also
        // what reconciles a directory switch: the run that registers a new
        // account replaces the `accounturi` naming the old one here, before
        // the order that record has to authorise is placed.
        let issuance = IssuancePolicy {
            issuer: environment.issuer_domain().to_owned(),
            account_uri: account.id().to_owned(),
            report_to: contact
                .iter()
                .filter(|c| c.starts_with("mailto:") || c.starts_with("https://"))
                .map(|c| (*c).to_owned())
                .collect(),
        };
        if let Err(error) = dns.authorize_issuer(zone, &issuance) {
            // Not fatal. A zone with no CAA record is the state every zone
            // starts in, and refusing to serve because the record could not be
            // written would turn a narrowing into a dependency. It is loud
            // because nothing else will notice, and because what the zone
            // already says is now what applies: no record leaves issuance
            // open to every authority, and a record naming another account
            // refuses the orders this run is about to place.
            tracing::warn!(
                zone,
                %error,
                account = account.id(),
                "could not publish this zone's CAA record; the record the zone already carries is \
                 the one that applies to every order this run places"
            );
        }

        let (certified, expiry, meta) =
            match stored_certificate(&store, zone, environment, &provider)? {
                Some(stored) => stored,
                None => {
                    let issued = acme::issue(&account, &domains, &dns).await?;
                    // Loaded before it is saved, so a pair that does not
                    // load is never written where the next boot reads it.
                    let certified =
                        certified_key_from_pem(&issued.cert_chain_pem, &issued.key_pem, &provider)?;
                    let expiry = leaf_expiry(&certified.cert)?;
                    let meta = CertMeta {
                        directory_url: environment.directory_url().to_owned(),
                        issued_at: OffsetDateTime::now_utc(),
                    };
                    store.save_certificate(&issued.cert_chain_pem, &issued.key_pem, &meta)?;
                    (certified, expiry, meta)
                }
            };

        let cert = SwappableCert::new(certified);
        let tracker = Mutex::new(RenewalTracker::new(expiry));

        tracing::info!(
            zone,
            directory = %meta.directory_url,
            expiry = %expiry,
            "tls certificate ready"
        );

        Ok(Arc::new(Self {
            store,
            dns,
            account,
            environment,
            domains,
            policy,
            tracker,
            cert,
            provider,
        }))
    }

    /// The zone this manager holds a certificate for.
    pub fn zone(&self) -> &str {
        // `domains[0]` is the apex; `zone_and_wildcard` builds it that way.
        &self.domains[0]
    }

    /// The live certificate resolver. Build the process's one
    /// `rustls::ServerConfig` from this once, at startup; renewal swaps the
    /// certificate this resolver hands back, not the config or the listener.
    pub fn resolver(&self) -> Arc<SwappableCert> {
        self.cert.clone()
    }

    /// How urgently the current renewal state should be surfaced, right now.
    pub async fn alert_level(&self) -> AlertLevel {
        self.tracker
            .lock()
            .await
            .alert_level(OffsetDateTime::now_utc())
    }

    /// Runs forever, checking on [`CHECK_INTERVAL`] whether renewal is due
    /// and attempting it when it is.
    ///
    /// A failed attempt never touches [`Self::resolver`]: the certificate
    /// already being served keeps being served, and the failure is only
    /// recorded in the tracker that [`Self::alert_level`] reads. Intended to
    /// be spawned once and left running for the life of the process.
    pub async fn run_renewal_loop(self: Arc<Self>) {
        let mut interval = tokio::time::interval(CHECK_INTERVAL);
        loop {
            interval.tick().await;
            self.maybe_renew().await;
        }
    }

    /// Checks whether renewal is due and, if so, attempts it once. Split out
    /// from [`Self::run_renewal_loop`] so a test can call it directly on a
    /// clock it controls, without waiting on [`CHECK_INTERVAL`].
    pub async fn maybe_renew(&self) {
        let now = OffsetDateTime::now_utc();
        let due = {
            let tracker = self.tracker.lock().await;
            tracker.due(now, &self.policy)
        };
        if !due {
            return;
        }

        tracing::info!(domains = ?self.domains, "renewal due; requesting a new certificate");
        match acme::issue(&self.account, &self.domains, &self.dns).await {
            Ok(issued) => self.apply_renewal(issued, now).await,
            Err(err) => self.record_failure(err, now).await,
        }
    }

    async fn apply_renewal(&self, issued: acme::IssuedCertificate, now: OffsetDateTime) {
        let certified = match certified_key_from_pem(
            &issued.cert_chain_pem,
            &issued.key_pem,
            &self.provider,
        ) {
            Ok(certified) => certified,
            Err(err) => {
                tracing::error!(%err, "renewed certificate could not be loaded; still serving the old one");
                self.tracker.lock().await.record_failure(now);
                return;
            }
        };
        let expiry = match leaf_expiry(&certified.cert) {
            Ok(expiry) => expiry,
            Err(err) => {
                tracing::error!(%err, "could not read the renewed certificate's expiry; still serving the old one");
                self.tracker.lock().await.record_failure(now);
                return;
            }
        };

        let meta = CertMeta {
            directory_url: self.environment.directory_url().to_owned(),
            issued_at: now,
        };
        if let Err(err) =
            self.store
                .save_certificate(&issued.cert_chain_pem, &issued.key_pem, &meta)
        {
            // The new certificate is good but unsaved: still swap it in, so
            // this process keeps serving a valid certificate, but say loudly
            // that a restart would lose it and re-issue.
            tracing::error!(%err, "renewed certificate could not be written to disk; serving it anyway");
        }

        // One atomic swap; see `resolver.rs` for why this and not two
        // separate fields.
        self.cert.swap(certified);
        self.tracker.lock().await.record_success(now, expiry);
        tracing::info!(%expiry, "renewed the tls certificate");
    }

    async fn record_failure(&self, err: IssueError, now: OffsetDateTime) {
        let mut tracker = self.tracker.lock().await;
        tracker.record_failure(now);
        let level = tracker.alert_level(now);
        let failures = tracker.consecutive_failures();
        let expiry = tracker.current_expiry();
        drop(tracker);
        // `zone` on every arm: `docs/deployment.md` promises a failure "is
        // logged with the zone, the failure count and the expiry", and a
        // multi-zone deployment is exactly the case where an operator cannot
        // tell which zone this line is about without it.
        let zone = self.zone();
        match level {
            AlertLevel::Ok => {
                tracing::warn!(zone, %err, failures, %expiry, "certificate renewal failed; still serving the current certificate");
            }
            AlertLevel::Warning => {
                tracing::error!(zone, %err, failures, %expiry, "certificate renewal has failed repeatedly; needs attention before it becomes an outage");
            }
            AlertLevel::Critical => {
                tracing::error!(zone, %err, failures, %expiry, "certificate renewal is failing close to expiry -- every agent stops resolving if this is not fixed before then");
            }
        }
    }
}