didbot_schema/format/

language.rs

//! The `language` string format: an IETF BCP 47 language tag.
//!
//! BCP 47 in full is a registry lookup, and this is deliberately not that. It
//! checks the grammar — a primary subtag, then subtags of one to eight
//! alphanumerics — and one semantic rule that a grammar check misses:
//! RFC 5646 §2.2.9 forbids a variant or extension singleton subtag from
//! appearing twice, so `de-DE-1901-1901` and `en-a-foo-a-bar` are well formed
//! and not valid tags. Upstream keeps those in `language_parse_invalid.txt`
//! for the same reason it separates datetime's calendar cases.
//!
//! The primary subtag is required to be two or three lowercase letters, which
//! is narrower than the RFC's ABNF: that also admits a four-letter reserved
//! range and five-to-eight-letter registered languages. The interop vectors
//! declare `jaja` invalid and `JA` invalid, and following the vectors here
//! costs nothing — no language in current use has a primary subtag outside
//! ISO 639-1 and 639-3.
//!
//! The two irregular openings are kept: `x-` starts a wholly private-use tag
//! and `i-` starts a grandfathered one, and both bypass the primary-subtag
//! rule because neither has one.

use std::collections::BTreeSet;

/// Longest any one subtag may be.
pub const MAX_SUBTAG_LENGTH: usize = 8;

/// Why a string is not a valid language tag.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum LanguageError {
    /// Empty, or a leading, trailing or doubled hyphen.
    #[error("language tag has an empty subtag")]
    EmptySubtag,
    /// A subtag over [`MAX_SUBTAG_LENGTH`] characters.
    #[error("language subtag {0:?} is over {MAX_SUBTAG_LENGTH} characters")]
    SubtagTooLong(String),
    /// A character outside `A-Za-z0-9-`.
    #[error("invalid character {0:?} in language tag")]
    InvalidCharacter(char),
    /// The primary subtag is not two or three lowercase letters, and the tag
    /// does not open with the private-use or grandfathered singleton.
    #[error("language primary subtag {0:?} is not two or three lowercase letters")]
    BadPrimarySubtag(String),
    /// `x` or `i` with nothing after it.
    #[error("language tag {0:?} is a bare singleton with no subtags")]
    BareSingleton(String),
    /// A variant or extension singleton subtag appears twice.
    #[error("language subtag {0:?} is repeated")]
    RepeatedSubtag(String),
}

/// Checks a string against the language tag rules.
///
/// ```
/// use didbot_schema::format::language;
/// assert!(language::validate("en").is_ok());
/// assert!(language::validate("pt-BR").is_ok());
/// assert!(language::validate("hy-Latn-IT-arevela").is_ok());
/// assert!(language::validate("i-navajo").is_ok());
/// assert!(language::validate("JA").is_err());            // primary subtag is lowercase
/// assert!(language::validate("de-DE-1901-1901").is_err()); // repeated variant
/// ```
pub fn validate(value: &str) -> Result<(), LanguageError> {
    if let Some(ch) = value
        .chars()
        .find(|ch| !(ch.is_ascii_alphanumeric() || *ch == '-'))
    {
        return Err(LanguageError::InvalidCharacter(ch));
    }
    let subtags: Vec<&str> = value.split('-').collect();
    for subtag in &subtags {
        if subtag.is_empty() {
            return Err(LanguageError::EmptySubtag);
        }
        if subtag.len() > MAX_SUBTAG_LENGTH {
            return Err(LanguageError::SubtagTooLong((*subtag).to_owned()));
        }
    }

    let primary = subtags[0];
    // `x-` is a wholly private-use tag and `i-` a grandfathered one. Neither
    // carries a primary language subtag, and neither can carry a duplicate
    // that means anything, so both are accepted whole.
    if primary.eq_ignore_ascii_case("x") || primary.eq_ignore_ascii_case("i") {
        if subtags.len() < 2 {
            return Err(LanguageError::BareSingleton(primary.to_owned()));
        }
        return Ok(());
    }
    if !(matches!(primary.len(), 2 | 3) && primary.bytes().all(|b| b.is_ascii_lowercase())) {
        return Err(LanguageError::BadPrimarySubtag(primary.to_owned()));
    }

    // RFC 5646 §2.2.9: no variant subtag and no extension singleton may
    // repeat. Both comparisons are case-insensitive, because subtag case is
    // presentational only and `en-rozaj-ROZAJ` is the same tag twice.
    let mut seen: BTreeSet<String> = BTreeSet::new();
    for subtag in &subtags[1..] {
        if subtag.eq_ignore_ascii_case("x") {
            // Everything after the private-use singleton is opaque.
            break;
        }
        let is_singleton = subtag.len() == 1;
        // A variant is five to eight alphanumerics, or four beginning with a
        // digit. Two- and three-letter subtags are scripts, regions and
        // extended languages, which may legitimately look alike.
        let is_variant = subtag.len() >= 5
            || (subtag.len() == 4 && subtag.starts_with(|c: char| c.is_ascii_digit()));
        if (is_singleton || is_variant) && !seen.insert(subtag.to_ascii_lowercase()) {
            return Err(LanguageError::RepeatedSubtag((*subtag).to_owned()));
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn scripts_and_regions_may_look_alike_without_being_duplicates() {
        // Only variants and extension singletons are constrained; a script and
        // a region of the same spelling are different slots.
        assert!(validate("sgn-BE-NL").is_ok());
        assert!(validate("zh-Hant-HK").is_ok());
    }

    #[test]
    fn a_repeated_extension_singleton_is_refused_case_insensitively() {
        assert_eq!(
            validate("en-u-co-phonebk-U-ca-buddhist"),
            Err(LanguageError::RepeatedSubtag("U".to_owned()))
        );
    }

    #[test]
    fn private_use_subtags_are_opaque() {
        // Everything after `x` is by definition meaningless to us, so a
        // repeat inside it is not an error.
        assert!(validate("en-x-foo-foo").is_ok());
        assert!(validate("X-fr-CH").is_ok());
    }
}