sov-kernel-monster / frontend /wasm /src /unicode_engine.rs
SNAPKITTYWEST's picture
chore: push full sov-kernel-monster content from local build
9425aed verified
Raw
History Blame Contribute Delete
8.42 kB
/// Unicode Engine โ€” deterministic Unicode normalization and analysis
/// Provides NFC/NFKC normalization, code point analysis, bidirectional detection,
/// and roundtrip verification with zero external state.
use wasm_bindgen::prelude::*;
use unicode_normalization::UnicodeNormalization;
#[allow(unused_imports)]
use serde_json::{json, Value};
/// Normalize text to specified Unicode form (NFC or NFKC)
///
/// # Arguments
/// * `input` - The text to normalize
/// * `form` - "NFC" or "NFKC" (case-insensitive)
///
/// # Returns
/// Normalized UTF-8 string
#[wasm_bindgen]
pub fn normalize(input: &str, form: &str) -> String {
match form.to_uppercase().as_str() {
"NFC" => input.nfc().collect::<String>(),
"NFKC" => input.nfkc().collect::<String>(),
_ => input.to_string(), // Default: return as-is
}
}
/// Encode text to comprehensive Unicode representation
///
/// # Returns
/// JSON object with structure:
/// {
/// "normalized": "NFC-normalized text",
/// "codePoints": [array of decimal code points],
/// "utf8Bytes": [array of decimal byte values],
/// "length": number of Unicode scalar values,
/// "byteLength": byte count of UTF-8 encoding
/// }
#[wasm_bindgen]
pub fn encode(input: &str) -> String {
let normalized = input.nfc().collect::<String>();
let code_points: Vec<u32> = input.chars().map(|c| c as u32).collect();
let utf8_bytes: Vec<u8> = input.as_bytes().to_vec();
let length = input.chars().count();
let byte_length = utf8_bytes.len();
let result = json!({
"normalized": normalized,
"codePoints": code_points,
"utf8Bytes": utf8_bytes,
"length": length,
"byteLength": byte_length
});
result.to_string()
}
/// Detect whether input contains astral plane characters (code points > 0xFFFF)
///
/// Astral characters require surrogate pairs in UTF-16 and special handling in JS
#[wasm_bindgen]
pub fn has_astral_characters(input: &str) -> bool {
input.chars().any(|c| (c as u32) > 0xFFFF)
}
/// Detect combining marks (Unicode block U+0300โ€“U+036F, also U+1AB0โ€“U+1AFF, etc.)
///
/// Returns true if input contains any combining diacritical marks
#[wasm_bindgen]
pub fn has_combining_marks(input: &str) -> bool {
input.chars().any(|c| {
let code = c as u32;
// Combining Diacritical Marks: U+0300โ€“U+036F
(code >= 0x0300 && code <= 0x036F) ||
// Combining Diacritical Marks Extended: U+1AB0โ€“U+1AFF
(code >= 0x1AB0 && code <= 0x1AFF) ||
// Combining Diacritical Marks Supplement: U+1DC0โ€“U+1DFF
(code >= 0x1DC0 && code <= 0x1DFF) ||
// Combining Half Marks: U+FE20โ€“U+FE2F
(code >= 0xFE20 && code <= 0xFE2F)
})
}
/// Detect bidirectional text direction
///
/// Returns one of: "LTR" (left-to-right), "RTL" (right-to-left), "NEUTRAL"
#[wasm_bindgen]
pub fn detect_bidi_level(input: &str) -> String {
let mut has_rtl = false;
let mut has_ltr = false;
for c in input.chars() {
let code = c as u32;
// RTL ranges: Hebrew, Arabic, Syriac, Thaana, Devanagari (some), etc.
if (code >= 0x0590 && code <= 0x08FF) || // Hebrew + Arabic
(code >= 0xFB1D && code <= 0xFB4F) || // Hebrew presentation forms
(code >= 0xFB50 && code <= 0xFDFF) || // Arabic presentation forms A
(code >= 0xFE70 && code <= 0xFEFF) { // Arabic presentation forms B
has_rtl = true;
}
// LTR ranges: Latin, Greek, Cyrillic, etc.
if (code >= 0x0041 && code <= 0x005A) || // A-Z
(code >= 0x0061 && code <= 0x007A) || // a-z
(code >= 0x0391 && code <= 0x03C9) || // Greek
(code >= 0x0410 && code <= 0x044F) { // Cyrillic
has_ltr = true;
}
}
match (has_rtl, has_ltr) {
(true, false) => "RTL".to_string(),
(false, true) => "LTR".to_string(),
(true, true) => "MIXED".to_string(),
(false, false) => "NEUTRAL".to_string(),
}
}
/// Verify encodeโ†’decode roundtrip consistency
///
/// Encodes text, extracts code points, reconstructs, and checks equality
/// Returns true if roundtrip preserves original string
#[wasm_bindgen]
pub fn verify_roundtrip(input: &str) -> bool {
// Method 1: Direct encode/decode via code points
let encoded: String = input
.chars()
.filter_map(|c| char::from_u32(c as u32))
.collect();
if encoded != input {
return false;
}
// Method 2: NFC roundtrip
let nfc_form: String = input.nfc().collect();
let nfc_again: String = nfc_form.nfc().collect();
if nfc_form != nfc_again {
return false;
}
// Method 3: UTF-8 byte roundtrip
let bytes = input.as_bytes();
if let Ok(reconstructed) = std::str::from_utf8(bytes) {
reconstructed == input
} else {
false
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_normalize_nfc() {
// รฉ can be represented as single character or e + combining acute
let decomposed = "e\u{0301}"; // e + combining acute
let normalized = normalize(decomposed, "NFC");
let expected = "รฉ"; // single character
assert_eq!(normalized, expected);
}
#[test]
fn test_normalize_nfkc() {
let input = "๏ฌ"; // fi ligature U+FB01
let normalized = normalize(input, "NFKC");
assert_eq!(normalized, "fi");
}
#[test]
fn test_encode_ascii() {
let input = "hello";
let result = encode(input);
let parsed: Value = serde_json::from_str(&result).unwrap();
assert_eq!(parsed["normalized"], "hello");
assert_eq!(parsed["length"], 5);
assert_eq!(parsed["byteLength"], 5);
assert_eq!(
parsed["codePoints"].as_array().unwrap().len(),
5
);
}
#[test]
fn test_encode_emoji() {
let input = "๐Ÿ‘‹"; // waving hand emoji
let result = encode(input);
let parsed: Value = serde_json::from_str(&result).unwrap();
assert_eq!(parsed["length"], 1);
assert_eq!(parsed["byteLength"], 4); // UTF-8 encoding of U+1F44B
}
#[test]
fn test_has_astral_characters_true() {
assert!(has_astral_characters("๐Ÿ‘‹")); // Emoji is astral
assert!(has_astral_characters("๐•ณ๐–Š๐–‘๐–‘๐–”")); // Mathematical alphanumeric symbols
}
#[test]
fn test_has_astral_characters_false() {
assert!(!has_astral_characters("hello"));
assert!(!has_astral_characters("cafรฉ"));
assert!(!has_astral_characters("ๆ—ฅๆœฌ่ชž")); // CJK is within BMP
}
#[test]
fn test_has_combining_marks_true() {
assert!(has_combining_marks("e\u{0301}")); // Explicit combining mark (e + combining acute)
assert!(has_combining_marks("a\u{0308}")); // a with combining diaeresis
}
#[test]
fn test_has_combining_marks_false() {
assert!(!has_combining_marks("hello"));
assert!(!has_combining_marks("รฉ")); // Precomposed form (NFC) has no combining marks
}
#[test]
fn test_detect_bidi_ltr() {
let level = detect_bidi_level("Hello World");
assert_eq!(level, "LTR");
}
#[test]
fn test_detect_bidi_rtl() {
let level = detect_bidi_level("ืฉืœื•ื ืขื•ืœื"); // Hebrew
assert_eq!(level, "RTL");
}
#[test]
fn test_detect_bidi_mixed() {
let level = detect_bidi_level("Hello ืฉืœื•ื");
assert_eq!(level, "MIXED");
}
#[test]
fn test_detect_bidi_neutral() {
let level = detect_bidi_level("123 !@#");
assert_eq!(level, "NEUTRAL");
}
#[test]
fn test_verify_roundtrip_ascii() {
assert!(verify_roundtrip("hello"));
assert!(verify_roundtrip("The quick brown fox jumps over the lazy dog"));
}
#[test]
fn test_verify_roundtrip_unicode() {
assert!(verify_roundtrip("cafรฉ"));
assert!(verify_roundtrip("ๆ—ฅๆœฌ่ชž"));
assert!(verify_roundtrip("๐Ÿ‘‹๐ŸŒ"));
}
#[test]
fn test_verify_roundtrip_empty() {
assert!(verify_roundtrip(""));
}
#[test]
fn test_verify_roundtrip_complex() {
// Complex string with multiple scripts and astral characters
let complex = "Hello ู…ุฑุญุจุง ืฉืœื•ื ๆ—ฅๆœฌ่ชž ๐Ÿ‘‹";
assert!(verify_roundtrip(complex));
}
}