didbot_tls/

renew.rs

//! When to renew, and what a renewal that keeps failing looks like from the
//! outside.
//!
//! Everything in this module is pure: no network, no clock of its own — the
//! caller always supplies `now`. That is deliberate. "Renew well ahead of
//! expiry" and "a renewal that has been failing for days must not first
//! become visible as a total outage" are both decisions this module can get
//! wrong without ever touching a CA, so they are tested that way.

use time::{Duration, OffsetDateTime};

/// When renewal is attempted, and how often a failing attempt retries.
#[derive(Debug, Clone, Copy)]
pub struct RenewalPolicy {
    /// How long before expiry a healthy renewal cycle starts trying.
    ///
    /// Let's Encrypt suggests renewing at a third of a certificate's
    /// lifetime remaining; for a 90-day certificate that is 30 days, which
    /// is also comfortably longer than the retry ceiling below, so a renewal
    /// that fails immediately still gets many attempts before expiry.
    pub renew_before: Duration,
    /// How long to wait between retries once renewal is due and failing.
    ///
    /// Not exponential backoff: ACME rate limits key off account and order
    /// creation frequency, not raw request count, and a failure here is far
    /// more likely to be "the DNS credential rotated" or "the zone stopped
    /// answering" than transient CA load — the kind of failure a human fixes
    /// on their own schedule, not one that clears itself if retried faster.
    pub retry_interval: Duration,
}

impl Default for RenewalPolicy {
    fn default() -> Self {
        Self {
            renew_before: Duration::days(30),
            retry_interval: Duration::hours(1),
        }
    }
}

/// How urgently a failing renewal should be surfaced.
///
/// Three levels rather than a boolean because "renewal is failing" is not
/// the same fact on day one of a 30-day runway as it is on day twenty-nine —
/// the epic asks for exactly this distinction: a renewal that has been
/// failing quietly for days must not first become visible as a total
/// outage, which means the loudness has to climb *before* expiry, not at it.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum AlertLevel {
    /// Not due, or due and either succeeding or not yet retried enough times
    /// to distinguish transient trouble from a real problem.
    Ok,
    /// Renewal is due and has been failing long enough that a human should
    /// know, well before it is urgent.
    Warning,
    /// Renewal is failing and expiry is close enough that this is the last
    /// reasonable window to intervene before every agent stops resolving.
    Critical,
}

impl AlertLevel {
    /// The name this level carries outside the process — in `GET /health`'s
    /// body, and so in whatever alarm reads that body.
    ///
    /// A method rather than a `Serialize` derive because the wire spelling
    /// is a contract with an alarm expression, and one that changes when a
    /// variant is renamed is a contract that breaks silently. Spelled out
    /// here, a rename is a compile error at this match instead.
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Ok => "ok",
            Self::Warning => "warning",
            Self::Critical => "critical",
        }
    }
}

/// How close to expiry [`AlertLevel::Critical`] starts, regardless of how
/// many attempts have failed. An expired wildcard is a total outage, so this
/// floor does not wait for a retry count.
const CRITICAL_WINDOW: Duration = Duration::days(3);

/// How many consecutive failures, once renewal is due, promote `Ok` to
/// `Warning`. One is deliberately not enough — a single failed attempt an
/// hour after the last success is exactly what the retry interval exists to
/// absorb quietly; a pattern is what earns attention.
const WARNING_FAILURE_THRESHOLD: u32 = 3;

/// Tracks renewal attempts against one certificate's expiry, so that a
/// caller can ask two questions without touching the network: is a renewal
/// due right now, and how loudly should its recent history be surfaced.
#[derive(Debug, Clone)]
pub struct RenewalTracker {
    current_expiry: OffsetDateTime,
    last_attempt: Option<OffsetDateTime>,
    last_success: Option<OffsetDateTime>,
    consecutive_failures: u32,
}

impl RenewalTracker {
    /// A tracker for a certificate that expires at `current_expiry` and has
    /// no renewal history yet — the state right after loading a certificate
    /// from storage or completing first issuance.
    pub fn new(current_expiry: OffsetDateTime) -> Self {
        Self {
            current_expiry,
            last_attempt: None,
            last_success: None,
            consecutive_failures: 0,
        }
    }

    /// The expiry this tracker is currently measuring against.
    pub fn current_expiry(&self) -> OffsetDateTime {
        self.current_expiry
    }

    /// How many renewal attempts have failed since the last success (or
    /// since this tracker was created, if none has succeeded yet).
    pub fn consecutive_failures(&self) -> u32 {
        self.consecutive_failures
    }

    /// Records a successful renewal: resets the failure count and moves the
    /// tracked expiry to the new certificate's.
    pub fn record_success(&mut self, at: OffsetDateTime, new_expiry: OffsetDateTime) {
        self.last_attempt = Some(at);
        self.last_success = Some(at);
        self.consecutive_failures = 0;
        self.current_expiry = new_expiry;
    }

    /// Records a failed attempt. The expiry this tracker measures against is
    /// unchanged — the old certificate is still what is being served.
    pub fn record_failure(&mut self, at: OffsetDateTime) {
        self.last_attempt = Some(at);
        self.consecutive_failures += 1;
    }

    /// Whether a renewal attempt should be made now.
    ///
    /// True once inside the policy's renewal window and, if the last attempt
    /// failed, only again once `retry_interval` has passed — so a caller can
    /// poll this on a short, fixed tick without hammering the CA on every
    /// tick once renewal is due.
    pub fn due(&self, now: OffsetDateTime, policy: &RenewalPolicy) -> bool {
        if now < self.current_expiry - policy.renew_before {
            return false;
        }
        match self.last_attempt {
            None => true,
            Some(last) => now >= last + policy.retry_interval,
        }
    }

    /// How loudly the current state should be surfaced.
    pub fn alert_level(&self, now: OffsetDateTime) -> AlertLevel {
        let time_to_expiry = self.current_expiry - now;
        if time_to_expiry <= CRITICAL_WINDOW {
            // Inside the critical window, a certificate that has never even
            // been renewed yet at this expiry is not "fine" just because
            // nothing has failed — expiry itself is now the risk.
            return AlertLevel::Critical;
        }
        if self.consecutive_failures >= WARNING_FAILURE_THRESHOLD {
            return AlertLevel::Warning;
        }
        AlertLevel::Ok
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn t(days_from_now: i64) -> OffsetDateTime {
        OffsetDateTime::now_utc() + Duration::days(days_from_now)
    }

    #[test]
    fn not_due_far_from_expiry() {
        let tracker = RenewalTracker::new(t(60));
        let policy = RenewalPolicy::default();
        assert!(!tracker.due(OffsetDateTime::now_utc(), &policy));
    }

    #[test]
    fn due_inside_the_renewal_window() {
        let tracker = RenewalTracker::new(t(20));
        let policy = RenewalPolicy::default();
        assert!(tracker.due(OffsetDateTime::now_utc(), &policy));
    }

    #[test]
    fn a_fresh_failure_is_not_immediately_retried() {
        let mut tracker = RenewalTracker::new(t(20));
        let policy = RenewalPolicy::default();
        let now = OffsetDateTime::now_utc();
        tracker.record_failure(now);
        assert!(!tracker.due(now + Duration::minutes(1), &policy));
    }

    #[test]
    fn a_failure_is_retried_after_the_interval() {
        let mut tracker = RenewalTracker::new(t(20));
        let policy = RenewalPolicy::default();
        let now = OffsetDateTime::now_utc();
        tracker.record_failure(now);
        assert!(tracker.due(now + policy.retry_interval, &policy));
    }

    #[test]
    fn success_resets_failures_and_the_tracked_expiry() {
        let mut tracker = RenewalTracker::new(t(20));
        let now = OffsetDateTime::now_utc();
        tracker.record_failure(now);
        tracker.record_failure(now);
        let new_expiry = t(110);
        tracker.record_success(now, new_expiry);
        assert_eq!(tracker.consecutive_failures(), 0);
        assert_eq!(tracker.current_expiry(), new_expiry);
    }

    #[test]
    fn ok_while_healthy() {
        let tracker = RenewalTracker::new(t(60));
        assert_eq!(
            tracker.alert_level(OffsetDateTime::now_utc()),
            AlertLevel::Ok
        );
    }

    #[test]
    fn warning_after_repeated_failures_well_before_expiry() {
        let mut tracker = RenewalTracker::new(t(25));
        let now = OffsetDateTime::now_utc();
        tracker.record_failure(now);
        tracker.record_failure(now);
        tracker.record_failure(now);
        assert_eq!(tracker.alert_level(now), AlertLevel::Warning);
    }

    #[test]
    fn one_failure_alone_is_not_yet_a_warning() {
        let mut tracker = RenewalTracker::new(t(25));
        let now = OffsetDateTime::now_utc();
        tracker.record_failure(now);
        assert_eq!(tracker.alert_level(now), AlertLevel::Ok);
    }

    #[test]
    fn critical_inside_the_expiry_floor_even_with_no_failures_recorded() {
        // A tracker that was only just created (e.g. after a restart that
        // lost history) with an expiry two days out must not read as
        // healthy just because `consecutive_failures` happens to be zero.
        let tracker = RenewalTracker::new(t(2));
        assert_eq!(
            tracker.alert_level(OffsetDateTime::now_utc()),
            AlertLevel::Critical
        );
    }

    #[test]
    fn critical_overrides_warning_close_to_expiry() {
        let mut tracker = RenewalTracker::new(t(2));
        let now = OffsetDateTime::now_utc();
        tracker.record_failure(now);
        tracker.record_failure(now);
        tracker.record_failure(now);
        assert_eq!(tracker.alert_level(now), AlertLevel::Critical);
    }

    #[test]
    fn alert_levels_order_ok_below_warning_below_critical() {
        assert!(AlertLevel::Ok < AlertLevel::Warning);
        assert!(AlertLevel::Warning < AlertLevel::Critical);
    }
}