/// Unicode Engine — deterministic Unicode normalization and analysis /// Provides NFC/NFKC normalization, code point analysis, bidirectional detection, /// and roundtrip verification with zero external state. use wasm_bindgen::prelude::*; use unicode_normalization::UnicodeNormalization; #[allow(unused_imports)] use serde_json::{json, Value}; /// Normalize text to specified Unicode form (NFC or NFKC) /// /// # Arguments /// * `input` - The text to normalize /// * `form` - "NFC" or "NFKC" (case-insensitive) /// /// # Returns /// Normalized UTF-8 string #[wasm_bindgen] pub fn normalize(input: &str, form: &str) -> String { match form.to_uppercase().as_str() { "NFC" => input.nfc().collect::(), "NFKC" => input.nfkc().collect::(), _ => input.to_string(), // Default: return as-is } } /// Encode text to comprehensive Unicode representation /// /// # Returns /// JSON object with structure: /// { /// "normalized": "NFC-normalized text", /// "codePoints": [array of decimal code points], /// "utf8Bytes": [array of decimal byte values], /// "length": number of Unicode scalar values, /// "byteLength": byte count of UTF-8 encoding /// } #[wasm_bindgen] pub fn encode(input: &str) -> String { let normalized = input.nfc().collect::(); let code_points: Vec = input.chars().map(|c| c as u32).collect(); let utf8_bytes: Vec = input.as_bytes().to_vec(); let length = input.chars().count(); let byte_length = utf8_bytes.len(); let result = json!({ "normalized": normalized, "codePoints": code_points, "utf8Bytes": utf8_bytes, "length": length, "byteLength": byte_length }); result.to_string() } /// Detect whether input contains astral plane characters (code points > 0xFFFF) /// /// Astral characters require surrogate pairs in UTF-16 and special handling in JS #[wasm_bindgen] pub fn has_astral_characters(input: &str) -> bool { input.chars().any(|c| (c as u32) > 0xFFFF) } /// Detect combining marks (Unicode block U+0300–U+036F, also U+1AB0–U+1AFF, etc.) /// /// Returns true if input contains any combining diacritical marks #[wasm_bindgen] pub fn has_combining_marks(input: &str) -> bool { input.chars().any(|c| { let code = c as u32; // Combining Diacritical Marks: U+0300–U+036F (code >= 0x0300 && code <= 0x036F) || // Combining Diacritical Marks Extended: U+1AB0–U+1AFF (code >= 0x1AB0 && code <= 0x1AFF) || // Combining Diacritical Marks Supplement: U+1DC0–U+1DFF (code >= 0x1DC0 && code <= 0x1DFF) || // Combining Half Marks: U+FE20–U+FE2F (code >= 0xFE20 && code <= 0xFE2F) }) } /// Detect bidirectional text direction /// /// Returns one of: "LTR" (left-to-right), "RTL" (right-to-left), "NEUTRAL" #[wasm_bindgen] pub fn detect_bidi_level(input: &str) -> String { let mut has_rtl = false; let mut has_ltr = false; for c in input.chars() { let code = c as u32; // RTL ranges: Hebrew, Arabic, Syriac, Thaana, Devanagari (some), etc. if (code >= 0x0590 && code <= 0x08FF) || // Hebrew + Arabic (code >= 0xFB1D && code <= 0xFB4F) || // Hebrew presentation forms (code >= 0xFB50 && code <= 0xFDFF) || // Arabic presentation forms A (code >= 0xFE70 && code <= 0xFEFF) { // Arabic presentation forms B has_rtl = true; } // LTR ranges: Latin, Greek, Cyrillic, etc. if (code >= 0x0041 && code <= 0x005A) || // A-Z (code >= 0x0061 && code <= 0x007A) || // a-z (code >= 0x0391 && code <= 0x03C9) || // Greek (code >= 0x0410 && code <= 0x044F) { // Cyrillic has_ltr = true; } } match (has_rtl, has_ltr) { (true, false) => "RTL".to_string(), (false, true) => "LTR".to_string(), (true, true) => "MIXED".to_string(), (false, false) => "NEUTRAL".to_string(), } } /// Verify encode→decode roundtrip consistency /// /// Encodes text, extracts code points, reconstructs, and checks equality /// Returns true if roundtrip preserves original string #[wasm_bindgen] pub fn verify_roundtrip(input: &str) -> bool { // Method 1: Direct encode/decode via code points let encoded: String = input .chars() .filter_map(|c| char::from_u32(c as u32)) .collect(); if encoded != input { return false; } // Method 2: NFC roundtrip let nfc_form: String = input.nfc().collect(); let nfc_again: String = nfc_form.nfc().collect(); if nfc_form != nfc_again { return false; } // Method 3: UTF-8 byte roundtrip let bytes = input.as_bytes(); if let Ok(reconstructed) = std::str::from_utf8(bytes) { reconstructed == input } else { false } } #[cfg(test)] mod tests { use super::*; #[test] fn test_normalize_nfc() { // é can be represented as single character or e + combining acute let decomposed = "e\u{0301}"; // e + combining acute let normalized = normalize(decomposed, "NFC"); let expected = "é"; // single character assert_eq!(normalized, expected); } #[test] fn test_normalize_nfkc() { let input = "fi"; // fi ligature U+FB01 let normalized = normalize(input, "NFKC"); assert_eq!(normalized, "fi"); } #[test] fn test_encode_ascii() { let input = "hello"; let result = encode(input); let parsed: Value = serde_json::from_str(&result).unwrap(); assert_eq!(parsed["normalized"], "hello"); assert_eq!(parsed["length"], 5); assert_eq!(parsed["byteLength"], 5); assert_eq!( parsed["codePoints"].as_array().unwrap().len(), 5 ); } #[test] fn test_encode_emoji() { let input = "👋"; // waving hand emoji let result = encode(input); let parsed: Value = serde_json::from_str(&result).unwrap(); assert_eq!(parsed["length"], 1); assert_eq!(parsed["byteLength"], 4); // UTF-8 encoding of U+1F44B } #[test] fn test_has_astral_characters_true() { assert!(has_astral_characters("👋")); // Emoji is astral assert!(has_astral_characters("𝕳𝖊𝖑𝖑𝖔")); // Mathematical alphanumeric symbols } #[test] fn test_has_astral_characters_false() { assert!(!has_astral_characters("hello")); assert!(!has_astral_characters("café")); assert!(!has_astral_characters("日本語")); // CJK is within BMP } #[test] fn test_has_combining_marks_true() { assert!(has_combining_marks("e\u{0301}")); // Explicit combining mark (e + combining acute) assert!(has_combining_marks("a\u{0308}")); // a with combining diaeresis } #[test] fn test_has_combining_marks_false() { assert!(!has_combining_marks("hello")); assert!(!has_combining_marks("é")); // Precomposed form (NFC) has no combining marks } #[test] fn test_detect_bidi_ltr() { let level = detect_bidi_level("Hello World"); assert_eq!(level, "LTR"); } #[test] fn test_detect_bidi_rtl() { let level = detect_bidi_level("שלום עולם"); // Hebrew assert_eq!(level, "RTL"); } #[test] fn test_detect_bidi_mixed() { let level = detect_bidi_level("Hello שלום"); assert_eq!(level, "MIXED"); } #[test] fn test_detect_bidi_neutral() { let level = detect_bidi_level("123 !@#"); assert_eq!(level, "NEUTRAL"); } #[test] fn test_verify_roundtrip_ascii() { assert!(verify_roundtrip("hello")); assert!(verify_roundtrip("The quick brown fox jumps over the lazy dog")); } #[test] fn test_verify_roundtrip_unicode() { assert!(verify_roundtrip("café")); assert!(verify_roundtrip("日本語")); assert!(verify_roundtrip("👋🌍")); } #[test] fn test_verify_roundtrip_empty() { assert!(verify_roundtrip("")); } #[test] fn test_verify_roundtrip_complex() { // Complex string with multiple scripts and astral characters let complex = "Hello مرحبا שלום 日本語 👋"; assert!(verify_roundtrip(complex)); } }