Escape NFC_Quick_Check=Maybe characters in escape_debug_ext

Author: Jules-Bertholet

library/alloctests/tests/str.rs

@@ -1145,6 +1145,9 @@ fn test_escape_debug() {
     );
     assert_eq!("α\u{0340}".escape_debug().to_string(), "α\\u{0340}");
     assert_eq!("\u{0149}".escape_debug().to_string(), "\\u{0149}");
+    assert_eq!("\u{1100}\u{1161}".escape_debug().to_string(), "\u{1100}\\u{1161}");
+    assert_eq!("\u{1100}\u{11A8}".escape_debug().to_string(), "\u{1100}\\u{11A8}");
+    assert_eq!("\u{16D4C}\u{16D67}".escape_debug().to_string(), "\u{16D4C}\\u{16D67}");
 }
 
 #[test]

library/core/src/char/methods.rs

@@ -492,10 +492,15 @@ impl char {
             _ if self.is_control()
                 || self.is_private_use()
                 || self.is_whitespace()
-                || args.escape_grapheme_extender && self.is_grapheme_extender()
+                || args.escape_grapheme_extender_and_maybe_not_nfc
+                    && self.is_grapheme_extender()
                 || self.is_default_ignorable()
                 || self.is_format_control()
-                || self.is_full_composition_exclusion()
+                || match self.nfc_quick_check() {
+                    QuickCheckResult::No => true,
+                    QuickCheckResult::Maybe => args.escape_grapheme_extender_and_maybe_not_nfc,
+                    QuickCheckResult::Yes => false,
+                }
                 || self.is_deprecated()
                 || self.is_unassigned() =>
             {
@@ -1368,6 +1373,39 @@ impl char {
         !self.is_ascii() && unicode::Deprecated(self)
     }
 
+    /// Returns the value of the `NFC_Quick_Check` property for this character.
+    ///
+    /// For a character `c`, this method returns:
+    ///
+    /// - [`QuickCheckResult::No`] if `c.is_full_composition_exclusion()`.
+    ///   These characters cannot ever occur in an NFC-[normalized] string.
+    /// - [`QuickCheckResult::Maybe`] for characters that may occur in an NFC-[normalized] string,
+    ///   but may also be removed from a string by NFC [normalization], depending on context.
+    /// - [`QuickCheckResult::Yes`] for all other characters.
+    ///
+    /// [normalized]: https://www.unicode.org/faq/normalization.html
+    /// [normalization]: https://www.unicode.org/faq/normalization.html
+    ///
+    /// `NFC_Quick_Check` is [described] in Annex #15 ("Unicode Normalization Forms") of the Unicode Standard,
+    /// and [specified] in the Unicode Character Database [`DerivedNormalizationProps.txt`].
+    ///
+    /// [described]: https://www.unicode.org/reports/tr15/#Detecting_Normalization_Forms
+    /// [specified]: https://www.unicode.org/reports/tr44/#NFC_Quick_Check
+    /// [`DerivedNormalizationProps.txt`]: https://www.unicode.org/Public/UCD/latest/ucd/DerivedNormalizationProps.txt
+    #[must_use]
+    #[inline]
+    fn nfc_quick_check(self) -> QuickCheckResult {
+        if self.is_ascii() {
+            QuickCheckResult::Yes
+        } else if self.is_full_composition_exclusion() {
+            QuickCheckResult::No
+        } else if unicode::NFC_QC_Maybe(self) {
+            QuickCheckResult::Maybe
+        } else {
+            QuickCheckResult::Yes
+        }
+    }
+
     /// Returns an iterator that yields the lowercase mapping of this `char` as one or more
     /// `char`s.
     ///
@@ -2496,8 +2534,8 @@ impl char {
 }
 
 pub(crate) struct EscapeDebugExtArgs {
-    /// Escape Grapheme Extender codepoints?
-    pub(crate) escape_grapheme_extender: bool,
+    /// Escape codepoints with `Grapheme_Extend` or `NFC_Quick_Check=Maybe`?
+    pub(crate) escape_grapheme_extender_and_maybe_not_nfc: bool,
 
     /// Escape single quotes?
     pub(crate) escape_single_quote: bool,
@@ -2508,7 +2546,7 @@ pub(crate) struct EscapeDebugExtArgs {
 
 impl EscapeDebugExtArgs {
     pub(crate) const ESCAPE_ALL: Self = Self {
-        escape_grapheme_extender: true,
+        escape_grapheme_extender_and_maybe_not_nfc: true,
         escape_single_quote: true,
         escape_double_quote: true,
     };

library/core/src/char/mod.rs

@@ -674,3 +674,11 @@ pub enum CharCase {
     /// Uppercase. Corresponds to the `Uppercase` Unicode property.
     Upper = 0b11,
 }
+
+/// Returned by [`char::nfc_quick_check`].
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+enum QuickCheckResult {
+    No,
+    Maybe,
+    Yes,
+}

library/core/src/fmt/mod.rs

@@ -2943,7 +2943,7 @@ impl Debug for str {
             let mut chars = rest.chars();
             if let Some(c) = chars.next() {
                 let esc = c.escape_debug_ext(EscapeDebugExtArgs {
-                    escape_grapheme_extender: true,
+                    escape_grapheme_extender_and_maybe_not_nfc: true,
                     escape_single_quote: false,
                     escape_double_quote: true,
                 });
@@ -2975,7 +2975,7 @@ impl Debug for char {
     fn fmt(&self, f: &mut Formatter<'_>) -> Result {
         f.write_char('\'')?;
         let esc = self.escape_debug_ext(EscapeDebugExtArgs {
-            escape_grapheme_extender: true,
+            escape_grapheme_extender_and_maybe_not_nfc: true,
             escape_single_quote: true,
             escape_double_quote: false,
         });

library/core/src/str/lossy.rs

@@ -123,7 +123,7 @@ impl fmt::Debug for Debug<'_> {
                 let mut from = 0;
                 for (i, c) in valid.char_indices() {
                     let esc = c.escape_debug_ext(EscapeDebugExtArgs {
-                        escape_grapheme_extender: true,
+                        escape_grapheme_extender_and_maybe_not_nfc: true,
                         escape_single_quote: false,
                         escape_double_quote: true,
                     });

library/core/src/str/mod.rs

@@ -3263,7 +3263,7 @@ impl_fn_for_zst! {
     #[derive(Clone)]
     struct CharEscapeDebugContinue impl Fn = |c: char| -> char::EscapeDebug {
         c.escape_debug_ext(EscapeDebugExtArgs {
-            escape_grapheme_extender: false,
+            escape_grapheme_extender_and_maybe_not_nfc: false,
             escape_single_quote: true,
             escape_double_quote: true
         })

library/core/src/unicode/mod.rs

@@ -17,6 +17,7 @@ pub(crate) use unicode_data::grapheme_extend::lookup as Grapheme_Extend;
 pub(crate) use unicode_data::lowercase::lookup as Lowercase;
 pub(crate) use unicode_data::lt::lookup as Lt;
 pub(crate) use unicode_data::n::lookup as N;
+pub(crate) use unicode_data::nfc_qc_maybe::lookup as NFC_QC_Maybe;
 pub(crate) use unicode_data::uppercase::lookup as Uppercase;
 pub(crate) use unicode_data::white_space::lookup as White_Space;
 

library/core/src/unicode/unicode_data.rs

@@ -10,13 +10,14 @@
 // Lowercase                   :   943 bytes,   2569 codepoints in 676 ranges (U+0000AA - U+01E944) using bitset
 // Lt                          :    33 bytes,     31 codepoints in  10 ranges (U+0001C5 - U+001FFD) using skiplist
 // N                           :   463 bytes,   1914 codepoints in 145 ranges (U+0000B2 - U+01FBFA) using skiplist
+// NFC_QC_Maybe                :   155 bytes,    132 codepoints in  49 ranges (U+000300 - U+016D69) using skiplist
 // Uppercase                   :   799 bytes,   1980 codepoints in 659 ranges (U+0000C0 - U+01F18A) using bitset
 // White_Space                 :   256 bytes,     19 codepoints in   8 ranges (U+000085 - U+003001) using cascading
 // to_lower                    :  1112 bytes,   1462 codepoints in 185 ranges (U+0000C0 - U+01E921) using 2-level LUT
 // to_upper                    :  1998 bytes,   1554 codepoints in 299 ranges (U+0000B5 - U+01E943) using 2-level LUT
 // to_title                    :   340 bytes,    135 codepoints in  49 ranges (U+0000DF - U+00FB17) using 2-level LUT
 // to_casefold                 :    32 bytes,    174 codepoints in   5 ranges (U+000131 - U+00ABBF) using 2-level LUT
-// Total                       : 11756 bytes
+// Total                       : 11911 bytes
 
 #[inline(always)]
 const fn bitset_search<
@@ -889,6 +890,47 @@ pub mod n {
     }
 }
 
+#[rustfmt::skip]
+pub mod nfc_qc_maybe {
+    use super::ShortOffsetRunHeader;
+
+    static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 14] = [
+        ShortOffsetRunHeader::new(0, 768), ShortOffsetRunHeader::new(1, 1619),
+        ShortOffsetRunHeader::new(25, 2364), ShortOffsetRunHeader::new(27, 2878),
+        ShortOffsetRunHeader::new(33, 4142), ShortOffsetRunHeader::new(57, 4449),
+        ShortOffsetRunHeader::new(59, 6965), ShortOffsetRunHeader::new(63, 12441),
+        ShortOffsetRunHeader::new(65, 69818), ShortOffsetRunHeader::new(67, 70462),
+        ShortOffsetRunHeader::new(71, 71984), ShortOffsetRunHeader::new(93, 90398),
+        ShortOffsetRunHeader::new(95, 93543), ShortOffsetRunHeader::new(97, 1207657),
+    ];
+    static OFFSETS: [u8; 99] = [
+        0, 5, 1, 7, 2, 1, 1, 1, 1, 2, 6, 1, 7, 6, 4, 2, 1, 2, 6, 1, 9, 1, 2, 1, 0, 3, 0, 1, 129, 1,
+        24, 1, 0, 1, 23, 2, 102, 1, 24, 1, 126, 1, 107, 1, 18, 2, 103, 1, 24, 1, 114, 1, 4, 1, 15,
+        1, 0, 1, 0, 21, 50, 27, 0, 1, 0, 2, 0, 1, 108, 1, 0, 1, 24, 1, 96, 1, 2, 1, 6, 1, 2, 1, 1,
+        3, 230, 1, 9, 1, 2, 1, 241, 1, 0, 1, 0, 12, 0, 2, 0,
+    ];
+    #[inline]
+    pub fn lookup(c: char) -> bool {
+        debug_assert!(!c.is_ascii());
+        (c as u32) >= 0x300 && lookup_slow(c)
+    }
+
+    #[inline(never)]
+    fn lookup_slow(c: char) -> bool {
+        const {
+            assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
+            let mut i = 0;
+            while i < SHORT_OFFSET_RUNS.len() {
+                assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+                i += 1;
+            }
+        }
+        // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+        // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+        unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
+    }
+}
+
 #[rustfmt::skip]
 pub mod uppercase {
     static BITSET_CHUNKS_MAP: [u8; 125] = [

library/core/src/wtf8.rs

@@ -147,7 +147,7 @@ impl fmt::Debug for Wtf8 {
             use crate::fmt::Write as _;
             for c in s.chars().flat_map(|c| {
                 c.escape_debug_ext(EscapeDebugExtArgs {
-                    escape_grapheme_extender: true,
+                    escape_grapheme_extender_and_maybe_not_nfc: true,
                     escape_single_quote: false,
                     escape_double_quote: true,
                 })

library/coretests/tests/unicode.rs

@@ -122,6 +122,12 @@ fn n() {
     test_boolean_property(test_data::N, char::is_numeric);
 }
 
+#[test]
+#[cfg_attr(miri, ignore)] // Miri is too slow
+fn nfc_qc_maybe() {
+    test_boolean_property(test_data::NFC_QC_MAYBE, unicode_data::nfc_qc_maybe::lookup);
+}
+
 #[test]
 #[cfg_attr(miri, ignore)] // Miri is too slow
 fn uppercase() {