Upload 2 files

ChatIPC.cpp

@@ -309,7 +309,7 @@ static std::vector<std::string> tokenize_whitespace(const std::string &s){
     return out;
 }
 
-// Static helper to check dictionary presence using your existing pos cache
+// Static helper to check dictionary presence using the existing pos cache
 static inline bool check_dictionary(const std::string &word) {
     return !dictionary_pos_for_token(word).empty();
 }
@@ -484,8 +484,78 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
     return std::nullopt;
 }
 
+// Helper arrays for number-to-words
+static const char* const ones[] = {"", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
+                                   "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen",
+                                   "seventeen", "eighteen", "nineteen"};
+static const char* const tens[] = {"", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"};
+static const char* const scales[] = {"", "thousand", "million", "billion", "trillion", "quadrillion",
+                                     "quintillion", "sextillion", "septillion", "octillion", "nonillion",
+                                     "decillion", "undecillion", "duodecillion", "tredecillion",
+                                     "quattuordecillion", "quindecillion", "sexdecillion", "septendecillion",
+                                     "octodecillion", "novemdecillion", "vigintillion", "googol"};
+
+static std::vector<std::string> digit_to_words(const std::string& s) {
+    std::vector<std::string> words;
+    if (s.empty()) return words;
+
+    // Strip leading zeros
+    size_t first_non_zero = s.find_first_not_of('0');
+    if (first_non_zero == std::string::npos) {
+        words.push_back("zero");
+        return words;
+    }
+    std::string num = s.substr(first_non_zero);
+
+    // If it's absurdly large (e.g. googolplex logic), fall back to single digits to prevent overflow
+    // and maintain semantic definitions of the numbers.
+    if (num.length() > 69) {
+        for (char c : num) {
+            words.push_back(ones[c - '0']);
+        }
+        return words;
+    }
+
+    int scale_idx = 0;
+    while (!num.empty()) {
+        int chunk_len = std::min<int>(3, num.length());
+        std::string chunk_str = num.substr(num.length() - chunk_len);
+        num = num.substr(0, num.length() - chunk_len);
+
+        int chunk = std::stoi(chunk_str);
+        if (chunk > 0) {
+            std::vector<std::string> chunk_words;
+            int h = chunk / 100;
+            int remainder = chunk % 100;
+
+            if (h > 0) {
+                chunk_words.push_back(ones[h]);
+                chunk_words.push_back("hundred");
+            }
+
+            if (remainder > 0) {
+                if (remainder < 20) {
+                    chunk_words.push_back(ones[remainder]);
+                } else {
+                    chunk_words.push_back(tens[remainder / 10]);
+                    if (remainder % 10 > 0) {
+                        chunk_words.push_back(ones[remainder % 10]);
+                    }
+                }
+            }
+
+            if (scale_idx > 0 && scale_idx < static_cast<int>(sizeof(scales)/sizeof(scales[0]))) {
+                chunk_words.push_back(scales[scale_idx]);
+            }
+
+            words.insert(words.begin(), chunk_words.begin(), chunk_words.end());
+        }
+        scale_idx++;
+    }
+    return words;
+}
+
 static std::vector<std::string> tokenize_others(const std::string &s) {
-    // Step A: Check if the entire input string already has a dictionary definition.
     if (check_dictionary(s)) {
         std::string lower_s;
         lower_s.reserve(s.size());
@@ -495,15 +565,29 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
         return { lower_s };
     }
 
+    // Convert pure digits to words
+    bool is_all_digits = !s.empty() && std::all_of(s.begin(), s.end(), [](unsigned char c) {
+        return std::isdigit(c) != 0;
+    });
+    if (is_all_digits) {
+        return digit_to_words(s);
+    }
+
     std::vector<std::string> out;
     std::string cur;
 
-    // Lambda to flush the accumulated token buffer, validate it, and add to output
     auto flush = [&]() {
         if (!cur.empty()) {
-            auto valid_base = get_valid_base_form(cur);
-            if (valid_base.has_value()) {
-                out.push_back(valid_base.value());
+            // Check numbers inside mixed tokens (e.g. "word123")
+            bool cur_all_digits = std::all_of(cur.begin(), cur.end(), [](unsigned char c) { return std::isdigit(c) != 0; });
+            if (cur_all_digits) {
+                auto words = digit_to_words(cur);
+                out.insert(out.end(), words.begin(), words.end());
+            } else {
+                auto valid_base = get_valid_base_form(cur);
+                if (valid_base.has_value()) {
+                    out.push_back(valid_base.value());
+                }
             }
             cur.clear();
         }
@@ -513,26 +597,42 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
         unsigned char uc = static_cast<unsigned char>(s[i]);
         char ch = static_cast<char>(uc);
 
-        // Split on spaces, underscores, hyphens, and slashes
-        if (ch == '_' || ch == '-' || ch == '/' || std::isspace(uc)) {
+        // Check for Operator/Symbol extraction
+        if (std::ispunct(uc) || (std::isprint(uc) && !std::isalnum(uc) && !std::isspace(uc))) {            flush();
+            std::string sym(1, ch);
+
+            // Check for double-character operators (==, <=, >=, !=, ++, --, &&, ||, <<, >>)
+            if (i + 1 < s.size()) {
+                unsigned char nxt_uc = static_cast<unsigned char>(s[i+1]);
+                if (std::ispunct(nxt_uc) || (std::isprint(nxt_uc) && !std::isalnum(nxt_uc) && !std::isspace(nxt_uc))) {                    std::string sym2 = sym + static_cast<char>(nxt_uc);
+                    if (check_dictionary(sym2)) {
+                        out.push_back(sym2);
+                        ++i;
+                        continue;
+                    }
+                }
+            }
+            if (check_dictionary(sym)) {
+                out.push_back(sym);
+            }
+            continue;
+        }
+
+        if (std::isspace(uc)) {
             flush();
             continue;
         }
 
         const bool is_upper = std::isupper(uc) != 0;
         bool camel_boundary = false;
-
         if (is_upper && !cur.empty()) {
             unsigned char prev = static_cast<unsigned char>(s[i - 1]);
             const bool prev_lower_or_digit = (std::islower(prev) != 0) || (std::isdigit(prev) != 0);
             const bool prev_upper = std::isupper(prev) != 0;
-            const bool next_lower =
-                (i + 1 < s.size()) && (std::islower(static_cast<unsigned char>(s[i + 1])) != 0);
-
+            const bool next_lower = (i + 1 < s.size()) && (std::islower(static_cast<unsigned char>(s[i + 1])) != 0);
             camel_boundary = prev_lower_or_digit || (prev_upper && next_lower);
         }
 
-        // Split on transitions between alphabetical characters and digits (e.g., "book5read")
         bool digit_boundary = false;
         if (!cur.empty()) {
             unsigned char prev = static_cast<unsigned char>(s[i - 1]);
@@ -553,7 +653,6 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
             flush();
         }
     }
-
     flush();
     return out;
 }
@@ -635,64 +734,87 @@ static void build_def_tokens_cache(){
     global_def_tokens_cache.clear();
     global_pos_cache.clear();
 
-    global_def_tokens_cache.reserve(global_dictionary_entries.size());
-    global_pos_cache.reserve(global_dictionary_entries.size());
+    const int num_entries = static_cast<int>(global_dictionary_entries.size());
 
-    for (const auto &entry : global_dictionary_entries){
-        // 1. Tokenize definitions once per entry to avoid redundant processing
-        std::vector<std::string> all_def_toks;
-        for (const auto &def : entry.definitions){
-            auto toks = tokenize_others(def);
-            all_def_toks.insert(all_def_toks.end(), toks.begin(), toks.end());
-        }
+    #pragma omp parallel
+    {
+        std::unordered_map<std::string, std::vector<std::string>> local_def_cache;
+        std::unordered_map<std::string, std::vector<std::string>> local_pos_cache;
 
-        std::string pos = normalize_pos_tag(entry.pos);
+        #pragma omp for schedule(guided) nowait
+        for (int i = 0; i < num_entries; ++i){
+            const auto &entry = global_dictionary_entries[i];
 
-        // 2. Split entry.word by semicolons or commas
-        std::vector<std::string> sub_words;
-        std::string current_sub;
-        for (char c : entry.word) {
-            if (c == ';' || c == ',') {
-                if (!current_sub.empty()) {
-                    sub_words.push_back(current_sub);
-                    current_sub.clear();
-                }
-            } else {
-                current_sub.push_back(c);
+            std::vector<std::string> all_def_toks;
+            for (const auto &def : entry.definitions){
+                auto toks = tokenize_others(def);
+                all_def_toks.insert(all_def_toks.end(), toks.begin(), toks.end());
             }
-        }
-        if (!current_sub.empty()) {
-            sub_words.push_back(current_sub);
-        }
 
-        // 3. Register each sub-word to the cache
-        for (size_t i = 0; i < sub_words.size(); ++i) {
-            const std::string key = normalize_dictionary_key(sub_words[i]);
-            if (key.empty()) continue;
+            std::string pos = normalize_pos_tag(entry.pos);
 
-            if (!pos.empty()) {
-                // Move on the final sub-word, copy otherwise
-                if (i == sub_words.size() - 1) {
-                    global_pos_cache[key].push_back(std::move(pos));
+            std::vector<std::string> sub_words;
+            std::string current_sub;
+            for (char c : entry.word) {
+                if (c == ';' || c == ',') {
+                    if (!current_sub.empty()) {
+                        sub_words.push_back(current_sub);
+                        current_sub.clear();
+                    }
                 } else {
-                    global_pos_cache[key].push_back(pos);
+                    current_sub.push_back(c);
+                }
+            }
+            if (!current_sub.empty()) sub_words.push_back(current_sub);
+
+            for (size_t j = 0; j < sub_words.size(); ++j) {
+                const std::string key = normalize_dictionary_key(sub_words[j]);
+                if (key.empty()) continue;
+
+                if (!pos.empty()) {
+                    if (j == sub_words.size() - 1) {
+                        local_pos_cache[key].push_back(std::move(pos));
+                    } else {
+                        local_pos_cache[key].push_back(pos);
+                    }
                 }
+                auto &defs = local_def_cache[key];
+                defs.insert(defs.end(), all_def_toks.begin(), all_def_toks.end());
             }
+        }
 
-            auto &defs = global_def_tokens_cache[key];
-            defs.insert(defs.end(), all_def_toks.begin(), all_def_toks.end());
+        #pragma omp critical
+        {
+            for (auto &pr : local_def_cache) {
+                auto &vec = global_def_tokens_cache[pr.first];
+                vec.insert(vec.end(), std::make_move_iterator(pr.second.begin()), std::make_move_iterator(pr.second.end()));
+            }
+            for (auto &pr : local_pos_cache) {
+                auto &vec = global_pos_cache[pr.first];
+                vec.insert(vec.end(), std::make_move_iterator(pr.second.begin()), std::make_move_iterator(pr.second.end()));
+            }
         }
     }
 
-    // 4. Keep existing sorting and deduplication logic intact
-    for (auto &pr : global_def_tokens_cache){
-        auto &v = pr.second;
+    // Extract keys to use OpenMP parallel-for deduplication over hash map structures
+    std::vector<std::string> def_keys;
+    def_keys.reserve(global_def_tokens_cache.size());
+    for (const auto &pr : global_def_tokens_cache) def_keys.push_back(pr.first);
+
+    #pragma omp parallel for schedule(dynamic)
+    for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(def_keys.size()); ++i){
+        auto &v = global_def_tokens_cache[def_keys[i]];
         std::sort(v.begin(), v.end());
         v.erase(std::unique(v.begin(), v.end()), v.end());
     }
 
-    for (auto &pr : global_pos_cache){
-        auto &v = pr.second;
+    std::vector<std::string> pos_keys;
+    pos_keys.reserve(global_pos_cache.size());
+    for (const auto &pr : global_pos_cache) pos_keys.push_back(pr.first);
+
+    #pragma omp parallel for schedule(dynamic)
+    for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(pos_keys.size()); ++i){
+        auto &v = global_pos_cache[pos_keys[i]];
         std::sort(v.begin(), v.end());
         v.erase(std::unique(v.begin(), v.end()), v.end());
     }